public virtual bool AddSnapshot(ISnapshot snapshot)
{
if (snapshot == null)
{
return(false);
}
Logger.Debug(Messages.AddingSnapshot, snapshot.StreamId, snapshot.BucketId, snapshot.StreamRevision);
try
{
BsonDocument mongoSnapshot = snapshot.ToMongoSnapshot(_serializer);
IMongoQuery query = Query.EQ(MongoShapshotFields.Id, mongoSnapshot[MongoShapshotFields.Id]);
UpdateBuilder update = Update.Set(MongoShapshotFields.Payload, mongoSnapshot[MongoShapshotFields.Payload]);
// Doing an upsert instead of an insert allows us to overwrite an existing snapshot and not get stuck with a
// stream that needs to be snapshotted because the insert fails and the SnapshotRevision isn't being updated.
PersistedSnapshots.Update(query, update, UpdateFlags.Upsert);
// More commits could have been made between us deciding that a snapshot is required and writing it so just
// resetting the Unsnapshotted count may be a little off. Adding snapshots should be a separate process so
// this is a good chance to make sure the numbers are still in-sync - it only adds a 'read' after all ...
BsonDocument streamHeadId = GetStreamHeadId(snapshot.BucketId, snapshot.StreamId);
StreamHead streamHead = PersistedStreamHeads.FindOneById(streamHeadId).ToStreamHead();
int unsnapshotted = streamHead.HeadRevision - snapshot.StreamRevision;
PersistedStreamHeads.Update(
Query.EQ(MongoStreamHeadFields.Id, streamHeadId),
Update.Set(MongoStreamHeadFields.SnapshotRevision, snapshot.StreamRevision).Set(MongoStreamHeadFields.Unsnapshotted, unsnapshotted));
return(true);
}
catch (Exception)
{
return(false);
}
}
//public static (IEnumerable<EventData> Events, long Version) LoadBulkEvents(
// this DaprClient client, string storeName,
// string streamName, long version, Dictionary<string, string> meta, StreamHead head)
//{
// var events = client
// .LoadAsyncBulkEventsAsync(storeName, streamName, version, meta, head)
// .ToEnumerable();
// return (events, events.LastOrDefault()?.Version ?? head.Version);
//}
public static async IAsyncEnumerable <EventData> LoadAsyncBulkEventsAsync(
this DaprClient client, string storeName,
string streamName, long version, Dictionary <string, string> meta, StreamHead head, int chunkSize = 20)
{
var keys = Enumerable
.Range(version == default ? 1 : (int)version, (int)(head.Version) + (version == default ? default : 1))
.Where(x => x <= head.Version)
.Select(x => Naming.StreamKey(streamName, x))
.ToList();
if (!keys.Any())
{
yield break;
}
foreach (var chunk in keys.BuildChunks(chunkSize))
{
var events = (await client.GetBulkStateAsync(storeName, chunk.ToArray(), null, metadata: meta))
.Select(x => JsonSerializer.Deserialize <EventData>(x.Value))
.OrderBy(x => x.Version);
foreach (var e in events)
{
yield return(e);
}
}
}
private async ValueTask <StreamHead> UpdateStreamHeadSnapshotRevisionAsync(StreamHead streamHead, long snapshotRevision, CancellationToken cancellation)
{
var id = streamHead.Id;
var bucketId = streamHead.BucketId;
var streamId = streamHead.StreamId;
StreamHead desired;
while (streamHead.SnapshotRevision < snapshotRevision)
{
desired = new StreamHead(bucketId,
streamId,
streamHead.HeadRevision,
snapshotRevision,
streamHead.DispatchedRevision,
streamHead.Version + 1);
if (await _database.CompareExchangeAsync(desired, streamHead, (left, right) => left.Version == right.Version, cancellation))
{
return(desired);
}
streamHead = await _database.GetOneAsync <StreamHead>(p => p.Id == id, cancellation);
if (streamHead == null && (streamHead = await AddStreamHeadAsync(bucketId, streamId, cancellation)) == null)
{
return(null);
}
}
return(streamHead);
}
public virtual void Commit(Commit attempt)
{
ThrowWhenDisposed();
Logger.Debug(Resources.AttemptingToCommit, attempt.CommitId, attempt.StreamId, attempt.CommitSequence);
lock (_commits)
{
if (_commits.Contains(attempt))
{
throw new DuplicateCommitException();
}
if (_commits.Any(c => c.StreamId == attempt.StreamId && c.StreamRevision == attempt.StreamRevision))
{
throw new ConcurrencyException();
}
_stamps[attempt.CommitId] = attempt.CommitStamp;
_commits.Add(attempt);
_undispatched.Add(attempt);
StreamHead head = _heads.FirstOrDefault(x => x.StreamId == attempt.StreamId);
_heads.Remove(head);
Logger.Debug(Resources.UpdatingStreamHead, attempt.StreamId);
int snapshotRevision = head == null ? 0 : head.SnapshotRevision;
_heads.Add(new StreamHead(attempt.StreamId, attempt.StreamRevision, snapshotRevision));
}
}
private ValueTask <StreamHead> AddStreamHeadAsync(string bucketId,
string streamId,
long headRevision,
long snapshotRevision,
long dispatchedRevision,
CancellationToken cancellation)
{
var streamHead = new StreamHead(bucketId, streamId, headRevision, snapshotRevision, dispatchedRevision, version: 1);
return(_database.GetOrAdd(streamHead, cancellation));
}
public static async Task StateTransactionSliceAsync(this DaprClient client,
string storeName,
string streamHeadKey, StreamHead head, string headetag, Dictionary <string, string> meta, EventData[] versionedEvents, string sliceKey, string sliceetag)
{
var sliceReq = new StateTransactionRequest(sliceKey, JsonSerializer.SerializeToUtf8Bytes(versionedEvents), Client.StateOperationType.Upsert, string.IsNullOrWhiteSpace(sliceetag) ? null : sliceetag, metadata: meta);
var headReq = new StateTransactionRequest(streamHeadKey, JsonSerializer.SerializeToUtf8Bytes(head), Client.StateOperationType.Upsert, etag: string.IsNullOrWhiteSpace(headetag) ? null : headetag, metadata: meta);
var reqs = new List <StateTransactionRequest> {
sliceReq, headReq
};
await client.ExecuteStateTransactionAsync(storeName, reqs, meta);
}
public async Task <long> AppendToStreamAsync(string streamName, Action <StreamHead> concurrencyGuard, params EventData[] events)
{
var streamKey = $"{streamName}|head";
var(head, headetag) = await client.GetStateAndETagAsync <StreamHead>(StoreName, streamKey);
if (head == null)
{
head = new StreamHead();
}
if (!events.Any())
{
return(head.Version);
}
concurrencyGuard(head);
var newVersion = head.Version + events.Length;
var versionedEvents = events
.Select((e, i) => new EventData {
EventId = e.EventId, EventName = e.EventName, Data = e.Data, Version = head.Version + (i + 1)
})
.ToArray();
var sliceKey = $"{streamName}|{newVersion}";
var(slice, sliceetag) = await client.GetStateAndETagAsync <EventData[]>(StoreName, sliceKey);
if (slice != null)
{
throw new DBConcurrencyException($"Event slice {sliceKey} ending with event version {newVersion} already exists");
}
var sliceWriteSuccess = await client.TrySaveStateAsync(StoreName, sliceKey, versionedEvents, sliceetag);
if (!sliceWriteSuccess)
{
throw new DBConcurrencyException($"Error writing events. Event slice {sliceKey} ending with event version {newVersion} already exists");
}
head.Version = newVersion;
var headWriteSuccess = await client.TrySaveStateAsync(StoreName, streamKey, head, headetag);
if (!headWriteSuccess)
{
throw new DBConcurrencyException($"stream head {streamKey} have been updated");
}
return(newVersion);
}
public bool AddSnapshot(Snapshot snapshot)
{
lock (_commits)
{
StreamHead currentHead = _heads.FirstOrDefault(h => h.StreamId == snapshot.StreamId);
if (currentHead == null)
{
return(false);
}
_snapshots.Add(snapshot);
_heads.Remove(currentHead);
_heads.Add(new StreamHead(currentHead.BucketId, currentHead.StreamId, currentHead.HeadRevision, snapshot.StreamRevision));
}
return(true);
}
public async Task <long> AppendToStreamAsync(string streamName, Action <StreamHead> concurrencyGuard, params EventData[] events)
{
var streamHeadKey = Naming.StreamHead(streamName);
var meta = MetaProvider(streamName);
var(head, headetag) = await client.GetStateAndETagAsync <StreamHead>(StoreName, streamHeadKey, metadata : meta);
if (head == null)
{
head = new StreamHead();
}
if (!events.Any())
{
return(head.Version);
}
concurrencyGuard(head);
var newVersion = head.Version + events.Length;
var versionedEvents = events
.Select((e, i) => new EventData(e.EventId, e.EventName, e.Data, head.Version + (i + 1)))
.ToArray();
var sliceKey = Naming.StreamKey(streamName, newVersion);
var(slice, sliceetag) = await client.GetStateAndETagAsync <EventData[]>(StoreName, sliceKey, metadata : meta);
if (slice != null)
{
throw new DBConcurrencyException($"Event slice {sliceKey} ending with event version {newVersion} already exists");
}
head = new StreamHead(newVersion);
var task = Mode switch
{
SliceMode.Off => client.StateTransactionAsync(StoreName, streamName, streamHeadKey, head, headetag, meta, versionedEvents),
SliceMode.Transactional => client.StateTransactionSliceAsync(StoreName, streamHeadKey, head, headetag, meta, versionedEvents, sliceKey, sliceetag),
SliceMode.TwoPhased => client.TwoPhasedAsync(StoreName, streamHeadKey, head, headetag, meta, newVersion, versionedEvents, sliceKey, sliceetag),
_ => throw new Exception("Mode not supported")
};
await task;
return(newVersion);
}
private async ValueTask <StreamHead> UpdateStreamHeadDispatchedRevisionAsync(StreamHead streamHead, long dispatchedRevision, CancellationToken cancellation)
{
var id = streamHead.Id;
var bucketId = streamHead.BucketId;
var streamId = streamHead.StreamId;
StreamHead desired;
while (streamHead.DispatchedRevision < dispatchedRevision)
{
var sequentialDispatchedRevision = dispatchedRevision;
if (streamHead.DispatchedRevision < dispatchedRevision - 1)
{
var commits = GetCommitsInternalAsync(bucketId, streamId, streamHead.DispatchedRevision + 1, dispatchedRevision);
var latestDispatchedCommit = await LatestDispatchedCommitAsync(commits, cancellation);
if (latestDispatchedCommit == null)
{
return(streamHead);
}
sequentialDispatchedRevision = latestDispatchedCommit.StreamRevision;
}
desired = new StreamHead(bucketId,
streamId,
streamHead.HeadRevision,
streamHead.SnapshotRevision,
sequentialDispatchedRevision,
streamHead.Version + 1);
if (await _database.CompareExchangeAsync(desired, streamHead, (left, right) => left.Version == right.Version, cancellation))
{
return(desired);
}
streamHead = await _database.GetOneAsync <StreamHead>(p => p.Id == id, cancellation);
if (streamHead == null && (streamHead = await AddStreamHeadAsync(bucketId, streamId, cancellation)) == null)
{
return(null);
}
}
return(streamHead);
}
public static async Task TwoPhasedAsync(this DaprClient client,
string storeName,
string streamHeadKey, StreamHead head, string headetag, Dictionary <string, string> meta, long newVersion, EventData[] versionedEvents, string sliceKey, string sliceetag)
{
var sliceWriteSuccess = await client.TrySaveStateAsync(storeName, sliceKey, versionedEvents, sliceetag, metadata : meta);
if (!sliceWriteSuccess)
{
throw new DBConcurrencyException($"Error writing events. Event slice {sliceKey} ending with event version {newVersion} already exists");
}
var headWriteSuccess = await client.TrySaveStateAsync(storeName, streamHeadKey, head, headetag, metadata : meta);
if (!headWriteSuccess)
{
throw new DBConcurrencyException($"stream head {streamHeadKey} have been updated");
}
}
public virtual bool AddSnapshot(Snapshot snapshot)
{
ThrowWhenDisposed();
Logger.Debug(Resources.AddingSnapshot, snapshot.StreamId, snapshot.StreamRevision);
lock (_commits)
{
StreamHead currentHead = _heads.FirstOrDefault(h => h.StreamId == snapshot.StreamId);
if (currentHead == null)
{
return(false);
}
_snapshots.Add(snapshot);
_heads.Remove(currentHead);
_heads.Add(new StreamHead(currentHead.StreamId, currentHead.HeadRevision, snapshot.StreamRevision));
}
return(true);
}
public static async Task <(IEnumerable <EventData> Events, long Version)> LoadSlicesAsync(
this DaprClient client, string storeName, ILogger logger,
string streamName, long version, Dictionary <string, string> meta, StreamHead head)
{
var eventSlices = new List <EventData[]>();
using (logger.BeginScope("Loading head {streamName}. Starting version {version}", streamName, head.Version))
{
var next = head.Version;
while (next != 0 && next >= version)
{
var sliceKey = Naming.StreamKey(streamName, next);
var slice = await client.GetStateAsync <EventData[]>(storeName, sliceKey, metadata : meta);
logger.LogDebug("Slice {sliceKey} loaded range : {firstVersion} - {lastVersion}", sliceKey, slice.First().Version, slice.Last().Version);
next = slice.First().Version - 1;
if (next < version)
{
logger.LogDebug("Version within slice. Next : {next}. Version : {version}", next, version);
eventSlices.Add(slice.Where(e => e.Version >= version).ToArray());
break;
}
logger.LogDebug("Adding slice. Next : {next}. Version : {version}", next, version);
eventSlices.Add(slice);
}
logger.LogDebug("Done reading. Got {sliceCount} slices.", eventSlices.Count);
var events = eventSlices
.Reverse <EventData[]>()
.SelectMany(e => e)
.ToArray();
return(events, events.LastOrDefault()?.Version ?? head.Version);
}
}
public static async Task StateTransactionAsync(this DaprClient client,
string storeName,
string streamName, string streamHeadKey, StreamHead head, string headetag, Dictionary <string, string> meta, EventData[] versionedEvents)
{
var eventsReq = versionedEvents.Select(x => new StateTransactionRequest(Naming.StreamKey(streamName, x.Version), JsonSerializer.SerializeToUtf8Bytes(x), StateOperationType.Upsert, metadata: meta));
var headReq = new StateTransactionRequest(streamHeadKey, JsonSerializer.SerializeToUtf8Bytes(head), Client.StateOperationType.Upsert, etag: string.IsNullOrWhiteSpace(headetag) ? null : headetag, metadata: meta);
var reqs = new List <StateTransactionRequest>();
reqs.AddRange(eventsReq);
reqs.Add(headReq);
await client.ExecuteStateTransactionAsync(storeName, reqs, meta);
}
