public virtual bool AddSnapshot(ISnapshot snapshot)
{
if (snapshot == null)
{
return(false);
}
Logger.Debug(Messages.AddingSnapshot, snapshot.StreamId, snapshot.BucketId, snapshot.StreamRevision);
try
{
BsonDocument mongoSnapshot = snapshot.ToMongoSnapshot(_serializer);
IMongoQuery query = Query.EQ(MongoShapshotFields.Id, mongoSnapshot[MongoShapshotFields.Id]);
UpdateBuilder update = Update.Set(MongoShapshotFields.Payload, mongoSnapshot[MongoShapshotFields.Payload]);
// Doing an upsert instead of an insert allows us to overwrite an existing snapshot and not get stuck with a
// stream that needs to be snapshotted because the insert fails and the SnapshotRevision isn't being updated.
PersistedSnapshots.Update(query, update, UpdateFlags.Upsert);
// More commits could have been made between us deciding that a snapshot is required and writing it so just
// resetting the Unsnapshotted count may be a little off. Adding snapshots should be a separate process so
// this is a good chance to make sure the numbers are still in-sync - it only adds a 'read' after all ...
BsonDocument streamHeadId = GetStreamHeadId(snapshot.BucketId, snapshot.StreamId);
StreamHead streamHead = PersistedStreamHeads.FindOneById(streamHeadId).ToStreamHead();
int unsnapshotted = streamHead.HeadRevision - snapshot.StreamRevision;
PersistedStreamHeads.Update(
Query.EQ(MongoStreamHeadFields.Id, streamHeadId),
Update.Set(MongoStreamHeadFields.SnapshotRevision, snapshot.StreamRevision).Set(MongoStreamHeadFields.Unsnapshotted, unsnapshotted));
return(true);
}
catch (Exception)
{
return(false);
}
}
private void UpdateStreamHeadAsync(string bucketId, string streamId, int streamRevision, int eventsCount)
{
ThreadPool.QueueUserWorkItem(x =>
{
try
{
TryMongo(() =>
{
BsonDocument streamHeadId = GetStreamHeadId(bucketId, streamId);
PersistedStreamHeads.Update(
Query.EQ(MongoStreamHeadFields.Id, streamHeadId),
Update
.Set(MongoStreamHeadFields.HeadRevision, streamRevision)
.Inc(MongoStreamHeadFields.SnapshotRevision, 0)
.Inc(MongoStreamHeadFields.Unsnapshotted, eventsCount),
UpdateFlags.Upsert);
});
}
catch (OutOfMemoryException ex)
{
throw;
}
catch (Exception ex)
{
//It is safe to ignore transient exception updating stream head.
Logger.Warn("Ignored Exception '{0}' when upserting the stream head Bucket Id [{1}] StreamId[{2}].\n {3}", ex.GetType().Name, bucketId, streamId, ex.ToString());
}
}, null
);
}
public virtual void Purge()
{
Logger.Warn(Messages.PurgingStorage);
PersistedCommits.RemoveAll();
PersistedStreamHeads.RemoveAll();
PersistedSnapshots.RemoveAll();
}
public virtual void Initialize()
{
if (Interlocked.Increment(ref _initialized) > 1)
{
return;
}
Logger.Debug(Messages.InitializingStorage);
TryMongo(() =>
{
PersistedCommits.EnsureIndex(IndexKeys.Ascending("Dispatched").Ascending(MongoFields.CommitStamp),
IndexOptions.SetName("Dispatched_Index").SetUnique(false));
PersistedCommits.EnsureIndex(IndexKeys.Ascending("_id.BucketId", "_id.StreamId", "Events.StreamRevision"),
IndexOptions.SetName("GetFrom_Index").SetUnique(true));
PersistedCommits.EnsureIndex(IndexKeys.Ascending(MongoFields.CommitStamp),
IndexOptions.SetName("CommitStamp_Index").SetUnique(false));
PersistedStreamHeads.EnsureIndex(IndexKeys.Ascending("Unsnapshotted"),
IndexOptions.SetName("Unsnapshotted_Index").SetUnique(false));
IMongoQuery query = Query.EQ("_id", MongoFields.CheckpointNumber);
IMongoUpdate update = Update.Replace(new BsonDocument {
{ "_id", "CheckpointNumber" }, { "seq", 0 }
});
Counters.Update(query, update, UpdateFlags.Upsert, WriteConcern.Acknowledged);
});
}
public void DeleteStream(string bucketId, string streamId)
{
Logger.Warn(Messages.DeletingStream, streamId, bucketId);
TryMongo(() =>
{
PersistedStreamHeads.Remove(
Query.EQ(MongoStreamHeadFields.Id, new BsonDocument {
{ MongoStreamHeadFields.BucketId, bucketId },
{ MongoStreamHeadFields.StreamId, streamId }
})
);
PersistedSnapshots.Remove(
Query.EQ(MongoShapshotFields.Id, new BsonDocument {
{ MongoShapshotFields.BucketId, bucketId },
{ MongoShapshotFields.StreamId, streamId }
})
);
PersistedCommits.Update(
Query.And(
Query.EQ(MongoCommitFields.BucketId, bucketId),
Query.EQ(MongoCommitFields.StreamId, streamId)
),
Update.Set(MongoCommitFields.BucketId, MongoSystemBuckets.RecycleBin),
UpdateFlags.Multi
);
});
}
private void UpdateStreamHeadInBackgroundThread(string bucketId, string streamId, int streamRevision, int eventsCount)
{
StartBackgroundThread(() =>
{
try
{
BsonDocument streamHeadId = GetStreamHeadId(bucketId, streamId);
PersistedStreamHeads.UpdateOne(
Builders <BsonDocument> .Filter.Eq(MongoStreamHeadFields.Id, streamHeadId),
Builders <BsonDocument> .Update
.Set(MongoStreamHeadFields.HeadRevision, streamRevision)
.Inc(MongoStreamHeadFields.SnapshotRevision, 0)
.Inc(MongoStreamHeadFields.Unsnapshotted, eventsCount),
new UpdateOptions()
{
IsUpsert = true
}
);
}
catch (OutOfMemoryException ex)
{
Logger.LogError(ex, "OutOfMemoryException:");
throw;
}
catch (Exception ex)
{
//It is safe to ignore transient exception updating stream head.
Logger.LogWarning(ex, "Ignored Exception '{exception}' when upserting the stream head Bucket Id [{id}] StreamId[{streamId}].\n", ex.GetType().Name, bucketId, streamId);
}
});
}
public void DeleteStream(string bucketId, string streamId)
{
if (Logger.IsWarnEnabled)
{
Logger.Warn(Messages.DeletingStream, streamId, bucketId);
}
TryMongo(() =>
{
PersistedStreamHeads.DeleteOne(
Builders <BsonDocument> .Filter.Eq(MongoStreamHeadFields.Id, new BsonDocument {
{ MongoStreamHeadFields.BucketId, bucketId },
{ MongoStreamHeadFields.StreamId, streamId }
})
);
PersistedSnapshots.DeleteMany(
Builders <BsonDocument> .Filter.Eq(MongoShapshotFields.Id, new BsonDocument {
{ MongoShapshotFields.BucketId, bucketId },
{ MongoShapshotFields.StreamId, streamId }
})
);
PersistedCommits.UpdateMany(
Builders <BsonDocument> .Filter.And(
Builders <BsonDocument> .Filter.Eq(MongoCommitFields.BucketId, bucketId),
Builders <BsonDocument> .Filter.Eq(MongoCommitFields.StreamId, streamId)
),
Builders <BsonDocument> .Update.Set(MongoCommitFields.BucketId, MongoSystemBuckets.RecycleBin)
);
});
}
public virtual void Initialize()
{
if (Interlocked.Increment(ref _initialized) > 1)
{
return;
}
Logger.Debug(Messages.InitializingStorage);
TryMongo(() =>
{
PersistedCommits.CreateIndex(
IndexKeys
.Ascending(MongoCommitFields.Dispatched)
.Ascending(MongoCommitFields.CommitStamp),
IndexOptions.SetName(MongoCommitIndexes.Dispatched).SetUnique(false)
);
PersistedCommits.CreateIndex(
IndexKeys.Ascending(
MongoCommitFields.BucketId,
MongoCommitFields.StreamId,
MongoCommitFields.StreamRevisionFrom,
MongoCommitFields.StreamRevisionTo
//,MongoCommitFields.FullqualifiedStreamRevision
),
IndexOptions.SetName(MongoCommitIndexes.GetFrom).SetUnique(true)
);
PersistedCommits.CreateIndex(
IndexKeys.Ascending(
MongoCommitFields.BucketId,
MongoCommitFields.StreamId,
MongoCommitFields.CommitSequence
),
IndexOptions.SetName(MongoCommitIndexes.LogicalKey).SetUnique(true)
);
PersistedCommits.CreateIndex(
IndexKeys.Ascending(MongoCommitFields.CommitStamp),
IndexOptions.SetName(MongoCommitIndexes.CommitStamp).SetUnique(false)
);
PersistedStreamHeads.CreateIndex(
IndexKeys.Ascending(MongoStreamHeadFields.Unsnapshotted),
IndexOptions.SetName(MongoStreamIndexes.Unsnapshotted).SetUnique(false)
);
if (_options.ServerSideOptimisticLoop)
{
PersistedCommits.Database.GetCollection("system.js").Save(new BsonDocument {
{ "_id", "insertCommit" },
{ "value", new BsonJavaScript(_options.GetInsertCommitScript()) }
});
}
EmptyRecycleBin();
});
}
public virtual void Purge()
{
Logger.Warn(Messages.PurgingStorage);
PersistedCommits.Drop();
PersistedStreamHeads.Drop();
PersistedSnapshots.Drop();
Counters.Drop();
}
/// <inheritdoc/>
public virtual void Purge()
{
Logger.LogWarning(Messages.PurgingStorage);
// @@review -> drop & create?
PersistedCommits.DeleteMany(Builders <BsonDocument> .Filter.Empty);
PersistedStreamHeads.DeleteMany(Builders <BsonDocument> .Filter.Empty);
PersistedSnapshots.DeleteMany(Builders <BsonDocument> .Filter.Empty);
}
public void DeleteStream(string bucketId, string streamId)
{
Logger.Warn(Messages.DeletingStream, streamId, bucketId);
TryMongo(() =>
{
PersistedStreamHeads.Remove(Query.And(Query.EQ("_id.BucketId", bucketId), Query.EQ("_id.StreamId", streamId)));
PersistedSnapshots.Remove(Query.And(Query.EQ("_id.BucketId", bucketId), Query.EQ("_id.StreamId", streamId)));
PersistedCommits.Remove(Query.And(Query.EQ("_id.BucketId", bucketId), Query.EQ("_id.StreamId", streamId)));
});
}
public void Purge(string bucketId)
{
Logger.Warn(Messages.PurgingBucket, bucketId);
TryMongo(() =>
{
PersistedStreamHeads.Remove(Query.EQ(MongoStreamHeadFields.FullQualifiedBucketId, bucketId));
PersistedSnapshots.Remove(Query.EQ(MongoShapshotFields.FullQualifiedBucketId, bucketId));
PersistedCommits.Remove(Query.EQ(MongoCommitFields.BucketId, bucketId));
});
}
public void Purge(string bucketId)
{
Logger.Warn(Messages.PurgingBucket, bucketId);
TryMongo(() =>
{
PersistedStreamHeads.Remove(Query.EQ("_id.BucketId", bucketId));
PersistedSnapshots.Remove(Query.EQ("_id.BucketId", bucketId));
PersistedCommits.Remove(Query.EQ("_id.BucketId", bucketId));
});
}
/// <inheritdoc/>
public void Purge(string bucketId)
{
Logger.LogWarning(Messages.PurgingBucket, bucketId);
TryMongo(() =>
{
PersistedStreamHeads.DeleteMany(Builders <BsonDocument> .Filter.Eq(MongoStreamHeadFields.FullQualifiedBucketId, bucketId));
PersistedSnapshots.DeleteMany(Builders <BsonDocument> .Filter.Eq(MongoShapshotFields.FullQualifiedBucketId, bucketId));
PersistedCommits.DeleteMany(Builders <BsonDocument> .Filter.Eq(MongoCommitFields.BucketId, bucketId));
});
}
public virtual IEnumerable <StreamHead> GetStreamsToSnapshot(int maxThreshold)
{
Logger.Debug(Messages.GettingStreamsToSnapshot);
return(TryMongo(() =>
{
IMongoQuery query = Query.GTE("Unsnapshotted", maxThreshold);
return PersistedStreamHeads.Find(query).SetSortOrder(SortBy.Descending("Unsnapshotted")).Select(x => x.ToStreamHead());
}));
}
private void UpdateStreamHeadAsync(string bucketId, string streamId, int streamRevision, int eventsCount)
{
ThreadPool.QueueUserWorkItem(x =>
TryMongo(() =>
{
BsonDocument streamHeadId = GetStreamHeadId(bucketId, streamId);
PersistedStreamHeads.Update(
Query.EQ(MongoStreamHeadFields.Id, streamHeadId),
Update
.Set(MongoStreamHeadFields.HeadRevision, streamRevision)
.Inc(MongoStreamHeadFields.SnapshotRevision, 0)
.Inc(MongoStreamHeadFields.Unsnapshotted, eventsCount),
UpdateFlags.Upsert);
}), null);
}
private void UpdateStreamHeadAsync(Guid streamId, int streamRevision, int eventsCount)
{
ThreadPool.QueueUserWorkItem(
x =>
TryMongo(
() =>
{
PersistedStreamHeads.Update(Query.EQ("_id", streamId),
Update.Set("HeadRevision", streamRevision)
.Inc("SnapshotRevision", 0)
.Inc("Unsnapshotted", eventsCount),
UpdateFlags.Upsert);
}),
null);
}
/// <inheritdoc/>
public virtual IEnumerable <IStreamHead> GetStreamsToSnapshot(string bucketId, int maxThreshold)
{
CheckIfSnapshotEnabled();
Logger.LogDebug(Messages.GettingStreamsToSnapshot);
return(TryMongo(() =>
{
var query = Builders <BsonDocument> .Filter.Gte(MongoStreamHeadFields.Unsnapshotted, maxThreshold);
return PersistedStreamHeads
.Find(query)
.Sort(Builders <BsonDocument> .Sort.Descending(MongoStreamHeadFields.Unsnapshotted))
.ToEnumerable()
.Select(x => x.ToStreamHead());
}));
}
private void UpdateStreamHeadAsync(string bucketId, string streamId, int streamRevision, int eventsCount)
{
ThreadPool.QueueUserWorkItem(x =>
TryMongo(() =>
{
try
{
BsonDocument streamHeadId = GetStreamHeadId(bucketId, streamId);
PersistedStreamHeads.Update(
Query.EQ(MongoStreamHeadFields.Id, streamHeadId),
Update
.Set(MongoStreamHeadFields.HeadRevision, streamRevision)
.Inc(MongoStreamHeadFields.SnapshotRevision, 0)
.Inc(MongoStreamHeadFields.Unsnapshotted, eventsCount),
UpdateFlags.Upsert);
}
catch (MongoDuplicateKeyException ex)
{
Logger.Warn("Duplicate key exception {0} when upserting the stream head {1} {2}.", ex, bucketId, streamId);
}
}), null);
}
public virtual void Initialize()
{
if (Interlocked.Increment(ref _initialized) > 1)
{
return;
}
Logger.Debug(Messages.InitializingStorage);
TryMongo(() =>
{
PersistedCommits.EnsureIndex(IndexKeys.Ascending("Dispatched").Ascending(MongoFields.CommitStamp),
IndexOptions.SetName("Dispatched_Index").SetUnique(false));
PersistedCommits.EnsureIndex(IndexKeys.Ascending("_id.BucketId", "_id.StreamId", "Events.StreamRevision"),
IndexOptions.SetName("GetFrom_Index").SetUnique(true));
PersistedCommits.EnsureIndex(IndexKeys.Ascending(MongoFields.CommitStamp), IndexOptions.SetName("CommitStamp_Index").SetUnique(false));
PersistedStreamHeads.EnsureIndex(IndexKeys.Ascending("Unsnapshotted"),
IndexOptions.SetName("Unsnapshotted_Index").SetUnique(false));
});
}
