public virtual bool AddSnapshot(ISnapshot snapshot) { if (snapshot == null) { return(false); } Logger.Debug(Messages.AddingSnapshot, snapshot.StreamId, snapshot.BucketId, snapshot.StreamRevision); try { BsonDocument mongoSnapshot = snapshot.ToMongoSnapshot(_serializer); IMongoQuery query = Query.EQ(MongoShapshotFields.Id, mongoSnapshot[MongoShapshotFields.Id]); UpdateBuilder update = Update.Set(MongoShapshotFields.Payload, mongoSnapshot[MongoShapshotFields.Payload]); // Doing an upsert instead of an insert allows us to overwrite an existing snapshot and not get stuck with a // stream that needs to be snapshotted because the insert fails and the SnapshotRevision isn't being updated. PersistedSnapshots.Update(query, update, UpdateFlags.Upsert); // More commits could have been made between us deciding that a snapshot is required and writing it so just // resetting the Unsnapshotted count may be a little off. Adding snapshots should be a separate process so // this is a good chance to make sure the numbers are still in-sync - it only adds a 'read' after all ... BsonDocument streamHeadId = GetStreamHeadId(snapshot.BucketId, snapshot.StreamId); StreamHead streamHead = PersistedStreamHeads.FindOneById(streamHeadId).ToStreamHead(); int unsnapshotted = streamHead.HeadRevision - snapshot.StreamRevision; PersistedStreamHeads.Update( Query.EQ(MongoStreamHeadFields.Id, streamHeadId), Update.Set(MongoStreamHeadFields.SnapshotRevision, snapshot.StreamRevision).Set(MongoStreamHeadFields.Unsnapshotted, unsnapshotted)); return(true); } catch (Exception) { return(false); } }
private void UpdateStreamHeadAsync(string bucketId, string streamId, int streamRevision, int eventsCount) { ThreadPool.QueueUserWorkItem(x => { try { TryMongo(() => { BsonDocument streamHeadId = GetStreamHeadId(bucketId, streamId); PersistedStreamHeads.Update( Query.EQ(MongoStreamHeadFields.Id, streamHeadId), Update .Set(MongoStreamHeadFields.HeadRevision, streamRevision) .Inc(MongoStreamHeadFields.SnapshotRevision, 0) .Inc(MongoStreamHeadFields.Unsnapshotted, eventsCount), UpdateFlags.Upsert); }); } catch (OutOfMemoryException ex) { throw; } catch (Exception ex) { //It is safe to ignore transient exception updating stream head. Logger.Warn("Ignored Exception '{0}' when upserting the stream head Bucket Id [{1}] StreamId[{2}].\n {3}", ex.GetType().Name, bucketId, streamId, ex.ToString()); } }, null ); }
public virtual void Purge() { Logger.Warn(Messages.PurgingStorage); PersistedCommits.RemoveAll(); PersistedStreamHeads.RemoveAll(); PersistedSnapshots.RemoveAll(); }
public virtual void Initialize() { if (Interlocked.Increment(ref _initialized) > 1) { return; } Logger.Debug(Messages.InitializingStorage); TryMongo(() => { PersistedCommits.EnsureIndex(IndexKeys.Ascending("Dispatched").Ascending(MongoFields.CommitStamp), IndexOptions.SetName("Dispatched_Index").SetUnique(false)); PersistedCommits.EnsureIndex(IndexKeys.Ascending("_id.BucketId", "_id.StreamId", "Events.StreamRevision"), IndexOptions.SetName("GetFrom_Index").SetUnique(true)); PersistedCommits.EnsureIndex(IndexKeys.Ascending(MongoFields.CommitStamp), IndexOptions.SetName("CommitStamp_Index").SetUnique(false)); PersistedStreamHeads.EnsureIndex(IndexKeys.Ascending("Unsnapshotted"), IndexOptions.SetName("Unsnapshotted_Index").SetUnique(false)); IMongoQuery query = Query.EQ("_id", MongoFields.CheckpointNumber); IMongoUpdate update = Update.Replace(new BsonDocument { { "_id", "CheckpointNumber" }, { "seq", 0 } }); Counters.Update(query, update, UpdateFlags.Upsert, WriteConcern.Acknowledged); }); }
public void DeleteStream(string bucketId, string streamId) { Logger.Warn(Messages.DeletingStream, streamId, bucketId); TryMongo(() => { PersistedStreamHeads.Remove( Query.EQ(MongoStreamHeadFields.Id, new BsonDocument { { MongoStreamHeadFields.BucketId, bucketId }, { MongoStreamHeadFields.StreamId, streamId } }) ); PersistedSnapshots.Remove( Query.EQ(MongoShapshotFields.Id, new BsonDocument { { MongoShapshotFields.BucketId, bucketId }, { MongoShapshotFields.StreamId, streamId } }) ); PersistedCommits.Update( Query.And( Query.EQ(MongoCommitFields.BucketId, bucketId), Query.EQ(MongoCommitFields.StreamId, streamId) ), Update.Set(MongoCommitFields.BucketId, MongoSystemBuckets.RecycleBin), UpdateFlags.Multi ); }); }
private void UpdateStreamHeadInBackgroundThread(string bucketId, string streamId, int streamRevision, int eventsCount) { StartBackgroundThread(() => { try { BsonDocument streamHeadId = GetStreamHeadId(bucketId, streamId); PersistedStreamHeads.UpdateOne( Builders <BsonDocument> .Filter.Eq(MongoStreamHeadFields.Id, streamHeadId), Builders <BsonDocument> .Update .Set(MongoStreamHeadFields.HeadRevision, streamRevision) .Inc(MongoStreamHeadFields.SnapshotRevision, 0) .Inc(MongoStreamHeadFields.Unsnapshotted, eventsCount), new UpdateOptions() { IsUpsert = true } ); } catch (OutOfMemoryException ex) { Logger.LogError(ex, "OutOfMemoryException:"); throw; } catch (Exception ex) { //It is safe to ignore transient exception updating stream head. Logger.LogWarning(ex, "Ignored Exception '{exception}' when upserting the stream head Bucket Id [{id}] StreamId[{streamId}].\n", ex.GetType().Name, bucketId, streamId); } }); }
public void DeleteStream(string bucketId, string streamId) { if (Logger.IsWarnEnabled) { Logger.Warn(Messages.DeletingStream, streamId, bucketId); } TryMongo(() => { PersistedStreamHeads.DeleteOne( Builders <BsonDocument> .Filter.Eq(MongoStreamHeadFields.Id, new BsonDocument { { MongoStreamHeadFields.BucketId, bucketId }, { MongoStreamHeadFields.StreamId, streamId } }) ); PersistedSnapshots.DeleteMany( Builders <BsonDocument> .Filter.Eq(MongoShapshotFields.Id, new BsonDocument { { MongoShapshotFields.BucketId, bucketId }, { MongoShapshotFields.StreamId, streamId } }) ); PersistedCommits.UpdateMany( Builders <BsonDocument> .Filter.And( Builders <BsonDocument> .Filter.Eq(MongoCommitFields.BucketId, bucketId), Builders <BsonDocument> .Filter.Eq(MongoCommitFields.StreamId, streamId) ), Builders <BsonDocument> .Update.Set(MongoCommitFields.BucketId, MongoSystemBuckets.RecycleBin) ); }); }
public virtual void Initialize() { if (Interlocked.Increment(ref _initialized) > 1) { return; } Logger.Debug(Messages.InitializingStorage); TryMongo(() => { PersistedCommits.CreateIndex( IndexKeys .Ascending(MongoCommitFields.Dispatched) .Ascending(MongoCommitFields.CommitStamp), IndexOptions.SetName(MongoCommitIndexes.Dispatched).SetUnique(false) ); PersistedCommits.CreateIndex( IndexKeys.Ascending( MongoCommitFields.BucketId, MongoCommitFields.StreamId, MongoCommitFields.StreamRevisionFrom, MongoCommitFields.StreamRevisionTo //,MongoCommitFields.FullqualifiedStreamRevision ), IndexOptions.SetName(MongoCommitIndexes.GetFrom).SetUnique(true) ); PersistedCommits.CreateIndex( IndexKeys.Ascending( MongoCommitFields.BucketId, MongoCommitFields.StreamId, MongoCommitFields.CommitSequence ), IndexOptions.SetName(MongoCommitIndexes.LogicalKey).SetUnique(true) ); PersistedCommits.CreateIndex( IndexKeys.Ascending(MongoCommitFields.CommitStamp), IndexOptions.SetName(MongoCommitIndexes.CommitStamp).SetUnique(false) ); PersistedStreamHeads.CreateIndex( IndexKeys.Ascending(MongoStreamHeadFields.Unsnapshotted), IndexOptions.SetName(MongoStreamIndexes.Unsnapshotted).SetUnique(false) ); if (_options.ServerSideOptimisticLoop) { PersistedCommits.Database.GetCollection("system.js").Save(new BsonDocument { { "_id", "insertCommit" }, { "value", new BsonJavaScript(_options.GetInsertCommitScript()) } }); } EmptyRecycleBin(); }); }
public virtual void Purge() { Logger.Warn(Messages.PurgingStorage); PersistedCommits.Drop(); PersistedStreamHeads.Drop(); PersistedSnapshots.Drop(); Counters.Drop(); }
/// <inheritdoc/> public virtual void Purge() { Logger.LogWarning(Messages.PurgingStorage); // @@review -> drop & create? PersistedCommits.DeleteMany(Builders <BsonDocument> .Filter.Empty); PersistedStreamHeads.DeleteMany(Builders <BsonDocument> .Filter.Empty); PersistedSnapshots.DeleteMany(Builders <BsonDocument> .Filter.Empty); }
public void DeleteStream(string bucketId, string streamId) { Logger.Warn(Messages.DeletingStream, streamId, bucketId); TryMongo(() => { PersistedStreamHeads.Remove(Query.And(Query.EQ("_id.BucketId", bucketId), Query.EQ("_id.StreamId", streamId))); PersistedSnapshots.Remove(Query.And(Query.EQ("_id.BucketId", bucketId), Query.EQ("_id.StreamId", streamId))); PersistedCommits.Remove(Query.And(Query.EQ("_id.BucketId", bucketId), Query.EQ("_id.StreamId", streamId))); }); }
public void Purge(string bucketId) { Logger.Warn(Messages.PurgingBucket, bucketId); TryMongo(() => { PersistedStreamHeads.Remove(Query.EQ(MongoStreamHeadFields.FullQualifiedBucketId, bucketId)); PersistedSnapshots.Remove(Query.EQ(MongoShapshotFields.FullQualifiedBucketId, bucketId)); PersistedCommits.Remove(Query.EQ(MongoCommitFields.BucketId, bucketId)); }); }
public void Purge(string bucketId) { Logger.Warn(Messages.PurgingBucket, bucketId); TryMongo(() => { PersistedStreamHeads.Remove(Query.EQ("_id.BucketId", bucketId)); PersistedSnapshots.Remove(Query.EQ("_id.BucketId", bucketId)); PersistedCommits.Remove(Query.EQ("_id.BucketId", bucketId)); }); }
/// <inheritdoc/> public void Purge(string bucketId) { Logger.LogWarning(Messages.PurgingBucket, bucketId); TryMongo(() => { PersistedStreamHeads.DeleteMany(Builders <BsonDocument> .Filter.Eq(MongoStreamHeadFields.FullQualifiedBucketId, bucketId)); PersistedSnapshots.DeleteMany(Builders <BsonDocument> .Filter.Eq(MongoShapshotFields.FullQualifiedBucketId, bucketId)); PersistedCommits.DeleteMany(Builders <BsonDocument> .Filter.Eq(MongoCommitFields.BucketId, bucketId)); }); }
public virtual IEnumerable <StreamHead> GetStreamsToSnapshot(int maxThreshold) { Logger.Debug(Messages.GettingStreamsToSnapshot); return(TryMongo(() => { IMongoQuery query = Query.GTE("Unsnapshotted", maxThreshold); return PersistedStreamHeads.Find(query).SetSortOrder(SortBy.Descending("Unsnapshotted")).Select(x => x.ToStreamHead()); })); }
private void UpdateStreamHeadAsync(string bucketId, string streamId, int streamRevision, int eventsCount) { ThreadPool.QueueUserWorkItem(x => TryMongo(() => { BsonDocument streamHeadId = GetStreamHeadId(bucketId, streamId); PersistedStreamHeads.Update( Query.EQ(MongoStreamHeadFields.Id, streamHeadId), Update .Set(MongoStreamHeadFields.HeadRevision, streamRevision) .Inc(MongoStreamHeadFields.SnapshotRevision, 0) .Inc(MongoStreamHeadFields.Unsnapshotted, eventsCount), UpdateFlags.Upsert); }), null); }
private void UpdateStreamHeadAsync(Guid streamId, int streamRevision, int eventsCount) { ThreadPool.QueueUserWorkItem( x => TryMongo( () => { PersistedStreamHeads.Update(Query.EQ("_id", streamId), Update.Set("HeadRevision", streamRevision) .Inc("SnapshotRevision", 0) .Inc("Unsnapshotted", eventsCount), UpdateFlags.Upsert); }), null); }
/// <inheritdoc/> public virtual IEnumerable <IStreamHead> GetStreamsToSnapshot(string bucketId, int maxThreshold) { CheckIfSnapshotEnabled(); Logger.LogDebug(Messages.GettingStreamsToSnapshot); return(TryMongo(() => { var query = Builders <BsonDocument> .Filter.Gte(MongoStreamHeadFields.Unsnapshotted, maxThreshold); return PersistedStreamHeads .Find(query) .Sort(Builders <BsonDocument> .Sort.Descending(MongoStreamHeadFields.Unsnapshotted)) .ToEnumerable() .Select(x => x.ToStreamHead()); })); }
private void UpdateStreamHeadAsync(string bucketId, string streamId, int streamRevision, int eventsCount) { ThreadPool.QueueUserWorkItem(x => TryMongo(() => { try { BsonDocument streamHeadId = GetStreamHeadId(bucketId, streamId); PersistedStreamHeads.Update( Query.EQ(MongoStreamHeadFields.Id, streamHeadId), Update .Set(MongoStreamHeadFields.HeadRevision, streamRevision) .Inc(MongoStreamHeadFields.SnapshotRevision, 0) .Inc(MongoStreamHeadFields.Unsnapshotted, eventsCount), UpdateFlags.Upsert); } catch (MongoDuplicateKeyException ex) { Logger.Warn("Duplicate key exception {0} when upserting the stream head {1} {2}.", ex, bucketId, streamId); } }), null); }
public virtual void Initialize() { if (Interlocked.Increment(ref _initialized) > 1) { return; } Logger.Debug(Messages.InitializingStorage); TryMongo(() => { PersistedCommits.EnsureIndex(IndexKeys.Ascending("Dispatched").Ascending(MongoFields.CommitStamp), IndexOptions.SetName("Dispatched_Index").SetUnique(false)); PersistedCommits.EnsureIndex(IndexKeys.Ascending("_id.BucketId", "_id.StreamId", "Events.StreamRevision"), IndexOptions.SetName("GetFrom_Index").SetUnique(true)); PersistedCommits.EnsureIndex(IndexKeys.Ascending(MongoFields.CommitStamp), IndexOptions.SetName("CommitStamp_Index").SetUnique(false)); PersistedStreamHeads.EnsureIndex(IndexKeys.Ascending("Unsnapshotted"), IndexOptions.SetName("Unsnapshotted_Index").SetUnique(false)); }); }