public IEnumerable <ICommit> GetFromTo(long from, long to)
{
if (Logger.IsDebugEnabled)
{
Logger.Debug(Messages.GettingCommitsFromToCheckpoint, from, to);
}
return(TryMongo(() =>
{
var filters = new List <FilterDefinition <BsonDocument> >()
{
Builders <BsonDocument> .Filter.Ne(MongoCommitFields.BucketId, MongoSystemBuckets.RecycleBin)
};
if (from > 0)
{
filters.Add(
Builders <BsonDocument> .Filter.Gt(MongoCommitFields.CheckpointNumber, from)
);
}
if (to < long.MaxValue)
{
filters.Add(
Builders <BsonDocument> .Filter.Lte(MongoCommitFields.CheckpointNumber, to)
);
}
var query = Builders <BsonDocument> .Filter.And(filters);
return PersistedCommits
.Find(query)
.Sort(SortByAscendingCheckpointNumber)
.ToEnumerable()
.Select(x => x.ToCommit(_serializer));
}));
}
public IEnumerable <ICommit> GetFrom(Int64 checkpointToken)
{
if (Logger.IsDebugEnabled)
{
Logger.Debug(Messages.GettingAllCommitsFromCheckpoint, checkpointToken);
}
return(TryMongo(() =>
{
var query = Builders <BsonDocument> .Filter.Ne(MongoCommitFields.BucketId, MongoSystemBuckets.RecycleBin);
if (checkpointToken > 0)
{
query = Builders <BsonDocument> .Filter.And(
query,
Builders <BsonDocument> .Filter.Gt(MongoCommitFields.CheckpointNumber, checkpointToken)
);
}
return PersistedCommits
.Find(query)
.Sort(SortByAscendingCheckpointNumber)
.ToEnumerable()
.Select(x => x.ToCommit(_serializer));
}));
}
public virtual IEnumerable <ICommit> GetFromTo(string bucketId, DateTime start, DateTime end)
{
if (Logger.IsDebugEnabled)
{
Logger.Debug(Messages.GettingAllCommitsFromTo, start, end, bucketId);
}
return(TryMongo(() =>
{
var filters = new List <FilterDefinition <BsonDocument> >()
{
Builders <BsonDocument> .Filter.Eq(MongoCommitFields.BucketId, bucketId)
};
if (start > DateTime.MinValue)
{
filters.Add(
Builders <BsonDocument> .Filter.Gte(MongoCommitFields.CommitStamp, start)
);
}
if (end < DateTime.MaxValue)
{
filters.Add(
Builders <BsonDocument> .Filter.Lt(MongoCommitFields.CommitStamp, end)
);
}
var query = Builders <BsonDocument> .Filter.And(filters);
return PersistedCommits
.Find(query)
.Sort(SortByAscendingCheckpointNumber)
.ToEnumerable()
.Select(x => x.ToCommit(_serializer));
}));
}
/// <inheritdoc/>
public virtual IEnumerable <ICommit> GetFrom(string bucketId, string streamId, int minRevision, int maxRevision)
{
Logger.LogDebug(Messages.GettingAllCommitsBetween, streamId, bucketId, minRevision, maxRevision);
return(TryMongo(() =>
{
var filters = new List <FilterDefinition <BsonDocument> >()
{
Builders <BsonDocument> .Filter.Eq(MongoCommitFields.BucketId, bucketId),
Builders <BsonDocument> .Filter.Eq(MongoCommitFields.StreamId, streamId)
};
if (minRevision > 0)
{
filters.Add(
Builders <BsonDocument> .Filter.Gte(MongoCommitFields.StreamRevisionTo, minRevision)
);
}
if (maxRevision < int.MaxValue)
{
filters.Add(
Builders <BsonDocument> .Filter.Lte(MongoCommitFields.StreamRevisionFrom, maxRevision)
);
}
var query = Builders <BsonDocument> .Filter.And(filters);
return PersistedCommits
.Find(query)
// .Sort(Builders<BsonDocument>.Sort.Ascending(MongoCommitFields.StreamRevisionFrom))
.Sort(SortByAscendingCheckpointNumber)
.ToEnumerable()
.Select(mc => mc.ToCommit(_serializer));
}));
}
public virtual void Initialize()
{
if (Interlocked.Increment(ref _initialized) > 1)
{
return;
}
Logger.Debug(Messages.InitializingStorage);
TryMongo(() =>
{
PersistedCommits.EnsureIndex(IndexKeys.Ascending("Dispatched").Ascending(MongoFields.CommitStamp),
IndexOptions.SetName("Dispatched_Index").SetUnique(false));
PersistedCommits.EnsureIndex(IndexKeys.Ascending("_id.BucketId", "_id.StreamId", "Events.StreamRevision"),
IndexOptions.SetName("GetFrom_Index").SetUnique(true));
PersistedCommits.EnsureIndex(IndexKeys.Ascending(MongoFields.CommitStamp),
IndexOptions.SetName("CommitStamp_Index").SetUnique(false));
PersistedStreamHeads.EnsureIndex(IndexKeys.Ascending("Unsnapshotted"),
IndexOptions.SetName("Unsnapshotted_Index").SetUnique(false));
IMongoQuery query = Query.EQ("_id", MongoFields.CheckpointNumber);
IMongoUpdate update = Update.Replace(new BsonDocument {
{ "_id", "CheckpointNumber" }, { "seq", 0 }
});
Counters.Update(query, update, UpdateFlags.Upsert, WriteConcern.Acknowledged);
});
}
public virtual void Commit(Commit attempt)
{
Logger.Debug(Messages.AttemptingToCommit, attempt.Events.Count, attempt.StreamId, attempt.CommitSequence);
TryMongo(() =>
{
BsonDocument commit = attempt.ToMongoCommit(_serializer);
try
{
// for concurrency / duplicate commit detection safe mode is required
PersistedCommits.Insert(commit, WriteConcern.Acknowledged);
UpdateStreamHeadAsync(attempt.BucketId, attempt.StreamId, attempt.StreamRevision, attempt.Events.Count);
Logger.Debug(Messages.CommitPersisted, attempt.CommitId);
}
catch (MongoException e)
{
if (!e.Message.Contains(ConcurrencyException))
{
throw;
}
Commit savedCommit = PersistedCommits.FindOne(attempt.ToMongoCommitIdQuery()).ToCommit(_serializer);
if (savedCommit.CommitId == attempt.CommitId)
{
throw new DuplicateCommitException();
}
Logger.Debug(Messages.ConcurrentWriteDetected);
throw new ConcurrencyException();
}
});
}
public virtual void Purge()
{
Logger.Warn(Messages.PurgingStorage);
PersistedCommits.RemoveAll();
PersistedStreamHeads.RemoveAll();
PersistedSnapshots.RemoveAll();
}
public void DeleteStream(string bucketId, string streamId)
{
Logger.Warn(Messages.DeletingStream, streamId, bucketId);
TryMongo(() =>
{
PersistedStreamHeads.Remove(
Query.EQ(MongoStreamHeadFields.Id, new BsonDocument {
{ MongoStreamHeadFields.BucketId, bucketId },
{ MongoStreamHeadFields.StreamId, streamId }
})
);
PersistedSnapshots.Remove(
Query.EQ(MongoShapshotFields.Id, new BsonDocument {
{ MongoShapshotFields.BucketId, bucketId },
{ MongoShapshotFields.StreamId, streamId }
})
);
PersistedCommits.Update(
Query.And(
Query.EQ(MongoCommitFields.BucketId, bucketId),
Query.EQ(MongoCommitFields.StreamId, streamId)
),
Update.Set(MongoCommitFields.BucketId, MongoSystemBuckets.RecycleBin),
UpdateFlags.Multi
);
});
}
public IEnumerable <ICommit> GetDeletedCommits()
{
return(TryMongo(() => PersistedCommits
.Find(Query.EQ(MongoCommitFields.BucketId, MongoSystemBuckets.RecycleBin))
.SetSortOrder(MongoCommitFields.CheckpointNumber)
.Select(mc => mc.ToCommit(_serializer))));
}
public virtual IEnumerable <ICommit> GetFrom(string bucketId, DateTime start)
{
if (Logger.IsDebugEnabled)
{
Logger.Debug(Messages.GettingAllCommitsFrom, start, bucketId);
}
return(TryMongo(() =>
{
var query = Builders <BsonDocument> .Filter.Eq(MongoCommitFields.BucketId, bucketId);
if (start != DateTime.MinValue)
{
query = Builders <BsonDocument> .Filter.And(
query,
Builders <BsonDocument> .Filter.Gte(MongoCommitFields.CommitStamp, start)
);
}
return PersistedCommits
.Find(query)
.Sort(SortByAscendingCheckpointNumber)
.ToEnumerable()
.Select(x => x.ToCommit(_serializer));
}));
}
public void DeleteStream(string bucketId, string streamId)
{
if (Logger.IsWarnEnabled)
{
Logger.Warn(Messages.DeletingStream, streamId, bucketId);
}
TryMongo(() =>
{
PersistedStreamHeads.DeleteOne(
Builders <BsonDocument> .Filter.Eq(MongoStreamHeadFields.Id, new BsonDocument {
{ MongoStreamHeadFields.BucketId, bucketId },
{ MongoStreamHeadFields.StreamId, streamId }
})
);
PersistedSnapshots.DeleteMany(
Builders <BsonDocument> .Filter.Eq(MongoShapshotFields.Id, new BsonDocument {
{ MongoShapshotFields.BucketId, bucketId },
{ MongoShapshotFields.StreamId, streamId }
})
);
PersistedCommits.UpdateMany(
Builders <BsonDocument> .Filter.And(
Builders <BsonDocument> .Filter.Eq(MongoCommitFields.BucketId, bucketId),
Builders <BsonDocument> .Filter.Eq(MongoCommitFields.StreamId, streamId)
),
Builders <BsonDocument> .Update.Set(MongoCommitFields.BucketId, MongoSystemBuckets.RecycleBin)
);
});
}
public virtual void Initialize()
{
if (Interlocked.Increment(ref _initialized) > 1)
{
return;
}
Logger.Debug(Messages.InitializingStorage);
TryMongo(() =>
{
PersistedCommits.CreateIndex(
IndexKeys
.Ascending(MongoCommitFields.Dispatched)
.Ascending(MongoCommitFields.CommitStamp),
IndexOptions.SetName(MongoCommitIndexes.Dispatched).SetUnique(false)
);
PersistedCommits.CreateIndex(
IndexKeys.Ascending(
MongoCommitFields.BucketId,
MongoCommitFields.StreamId,
MongoCommitFields.StreamRevisionFrom,
MongoCommitFields.StreamRevisionTo
//,MongoCommitFields.FullqualifiedStreamRevision
),
IndexOptions.SetName(MongoCommitIndexes.GetFrom).SetUnique(true)
);
PersistedCommits.CreateIndex(
IndexKeys.Ascending(
MongoCommitFields.BucketId,
MongoCommitFields.StreamId,
MongoCommitFields.CommitSequence
),
IndexOptions.SetName(MongoCommitIndexes.LogicalKey).SetUnique(true)
);
PersistedCommits.CreateIndex(
IndexKeys.Ascending(MongoCommitFields.CommitStamp),
IndexOptions.SetName(MongoCommitIndexes.CommitStamp).SetUnique(false)
);
PersistedStreamHeads.CreateIndex(
IndexKeys.Ascending(MongoStreamHeadFields.Unsnapshotted),
IndexOptions.SetName(MongoStreamIndexes.Unsnapshotted).SetUnique(false)
);
if (_options.ServerSideOptimisticLoop)
{
PersistedCommits.Database.GetCollection("system.js").Save(new BsonDocument {
{ "_id", "insertCommit" },
{ "value", new BsonJavaScript(_options.GetInsertCommitScript()) }
});
}
EmptyRecycleBin();
});
}
public IEnumerable <ICommit> GetDeletedCommits()
{
return(TryMongo(() => PersistedCommits
.Find(Builders <BsonDocument> .Filter.Eq(MongoCommitFields.BucketId, MongoSystemBuckets.RecycleBin))
.Sort(SortByAscendingCheckpointNumber)
.ToEnumerable()
.Select(mc => mc.ToCommit(_serializer))));
}
public virtual void Purge()
{
Logger.Warn(Messages.PurgingStorage);
PersistedCommits.Drop();
PersistedStreamHeads.Drop();
PersistedSnapshots.Drop();
Counters.Drop();
}
/// <inheritdoc/>
public virtual void Purge()
{
Logger.LogWarning(Messages.PurgingStorage);
// @@review -> drop & create?
PersistedCommits.DeleteMany(Builders <BsonDocument> .Filter.Empty);
PersistedStreamHeads.DeleteMany(Builders <BsonDocument> .Filter.Empty);
PersistedSnapshots.DeleteMany(Builders <BsonDocument> .Filter.Empty);
}
public virtual IEnumerable <Commit> GetFrom(DateTime start)
{
Logger.Debug(Messages.GettingAllCommitsFrom, start);
return
(TryMongo(
() =>
PersistedCommits.Find(Query.GTE("CommitStamp", start)).SetSortOrder("_id").Select(x => x.ToCommit(_serializer))));
}
public virtual IEnumerable <Commit> GetFromTo(string bucketId, DateTime start, DateTime end)
{
Logger.Debug(Messages.GettingAllCommitsFromTo, start, end);
return(TryMongo(() => PersistedCommits
.Find(Query.And(Query.GTE(MongoFields.CommitStamp, start), Query.LT(MongoFields.CommitStamp, end)))
.SetSortOrder(MongoFields.Id)
.Select(x => x.ToCommit(_serializer))));
}
public virtual IEnumerable <ICommit> GetUndispatchedCommits()
{
Logger.Debug(Messages.GettingUndispatchedCommits);
return(TryMongo(() => PersistedCommits
.Find(Query.EQ("Dispatched", false))
.SetSortOrder(MongoCommitFields.CheckpointNumber)
.Select(mc => mc.ToCommit(_serializer))));
}
public virtual IEnumerable <ICommit> GetFrom(string bucketId, DateTime start)
{
Logger.Debug(Messages.GettingAllCommitsFrom, start, bucketId);
return(TryMongo(() => PersistedCommits
.Find(Query.And(Query.EQ("_id.BucketId", bucketId), Query.GTE(MongoFields.CommitStamp, start)))
.SetSortOrder(MongoFields.CheckpointNumber)
.Select(x => x.ToCommit(_serializer))));
}
public IEnumerable <ICommit> GetFrom(string checkpointToken)
{
var intCheckpoint = IntCheckpoint.Parse(checkpointToken);
Logger.Debug(Messages.GettingAllCommitsFromCheckpoint, intCheckpoint.Value);
return(TryMongo(() => PersistedCommits
.Find(Query.GTE(MongoFields.CheckpointNumber, intCheckpoint.IntValue)))
.SetSortOrder(MongoFields.CheckpointNumber)
.Select(x => x.ToCommit(_serializer)));
}
/// <inheritdoc/>
public void Purge(string bucketId)
{
Logger.LogWarning(Messages.PurgingBucket, bucketId);
TryMongo(() =>
{
PersistedStreamHeads.DeleteMany(Builders <BsonDocument> .Filter.Eq(MongoStreamHeadFields.FullQualifiedBucketId, bucketId));
PersistedSnapshots.DeleteMany(Builders <BsonDocument> .Filter.Eq(MongoShapshotFields.FullQualifiedBucketId, bucketId));
PersistedCommits.DeleteMany(Builders <BsonDocument> .Filter.Eq(MongoCommitFields.BucketId, bucketId));
});
}
public void Purge(string bucketId)
{
Logger.Warn(Messages.PurgingBucket, bucketId);
TryMongo(() =>
{
PersistedStreamHeads.Remove(Query.EQ("_id.BucketId", bucketId));
PersistedSnapshots.Remove(Query.EQ("_id.BucketId", bucketId));
PersistedCommits.Remove(Query.EQ("_id.BucketId", bucketId));
});
}
public void Purge(string bucketId)
{
Logger.Warn(Messages.PurgingBucket, bucketId);
TryMongo(() =>
{
PersistedStreamHeads.Remove(Query.EQ(MongoStreamHeadFields.FullQualifiedBucketId, bucketId));
PersistedSnapshots.Remove(Query.EQ(MongoShapshotFields.FullQualifiedBucketId, bucketId));
PersistedCommits.Remove(Query.EQ(MongoCommitFields.BucketId, bucketId));
});
}
public void DeleteStream(string bucketId, string streamId)
{
Logger.Warn(Messages.DeletingStream, streamId, bucketId);
TryMongo(() =>
{
PersistedStreamHeads.Remove(Query.And(Query.EQ("_id.BucketId", bucketId), Query.EQ("_id.StreamId", streamId)));
PersistedSnapshots.Remove(Query.And(Query.EQ("_id.BucketId", bucketId), Query.EQ("_id.StreamId", streamId)));
PersistedCommits.Remove(Query.And(Query.EQ("_id.BucketId", bucketId), Query.EQ("_id.StreamId", streamId)));
});
}
private ICommit PersistWithServerSideOptimisticLoop(CommitAttempt attempt)
{
BsonDocument commitDoc = attempt.ToMongoCommit(_checkpointZero, _serializer);
return(TryMongo(() =>
{
var result = PersistedCommits.Database.Eval(new EvalArgs()
{
Code = _updateScript,
Lock = false,
Args = new BsonValue[] { commitDoc }
});
if (!result.IsBsonDocument)
{
throw new Exception("Invalid response. Check server side js");
}
if (result.AsBsonDocument.Contains("id"))
{
commitDoc["_id"] = result["id"];
UpdateStreamHeadAsync(attempt.BucketId, attempt.StreamId, attempt.StreamRevision, attempt.Events.Count);
Logger.Debug(Messages.CommitPersisted, attempt.CommitId);
}
else if (result.AsBsonDocument.Contains("err"))
{
var errorDocument = result.AsBsonDocument;
if (errorDocument["code"] != ConcurrencyExceptionCode)
{
throw new Exception(errorDocument["err"].AsString);
}
ICommit savedCommit = PersistedCommits.FindOne(attempt.ToMongoCommitIdQuery()).ToCommit(_serializer);
if (savedCommit.CommitId == attempt.CommitId)
{
throw new DuplicateCommitException(String.Format(
"Duplicated Commit: bucketId [{0}]: commitId [{1}] - streamId [{2}] - streamRevision [{3}]",
attempt.BucketId, attempt.CommitId, attempt.StreamId, attempt.StreamRevision));
}
Logger.Debug(Messages.ConcurrentWriteDetected);
throw new ConcurrencyException(String.Format(
"Concurrency exception forbucketId [{0}]: commitId [{1}] - streamId [{2}] - streamRevision [{3}]\n Inner provider error: {4}",
attempt.BucketId, attempt.CommitId, attempt.StreamId, attempt.StreamRevision, errorDocument["err"].AsString));
}
else
{
throw new Exception("Invalid response. Check server side js");
}
return commitDoc.ToCommit(_serializer);
}));
}
public virtual void MarkCommitAsDispatched(ICommit commit)
{
Logger.Debug(Messages.MarkingCommitAsDispatched, commit.CommitId);
TryMongo(() =>
{
IMongoQuery query = commit.ToMongoCommitIdQuery();
UpdateBuilder update = Update.Set(MongoCommitFields.Dispatched, true);
PersistedCommits.Update(query, update);
});
}
public void EmptyRecycleBin()
{
var lastCheckpointNumber = _getLastCheckPointNumber();
TryMongo(() =>
{
PersistedCommits.Remove(Query.And(
Query.EQ(MongoCommitFields.BucketId, MongoSystemBuckets.RecycleBin),
Query.LT(MongoCommitFields.CheckpointNumber, lastCheckpointNumber)
));
});
}
public void EmptyRecycleBin()
{
var lastCheckpointNumber = GetLastCommittedCheckPointNumber();
TryMongo(() =>
{
PersistedCommits.DeleteMany(Builders <BsonDocument> .Filter.And(
Builders <BsonDocument> .Filter.Eq(MongoCommitFields.BucketId, MongoSystemBuckets.RecycleBin),
Builders <BsonDocument> .Filter.Lt(MongoCommitFields.CheckpointNumber, lastCheckpointNumber)
));
});
}
public virtual IEnumerable <Commit> GetFrom(Guid streamId, int minRevision, int maxRevision)
{
Logger.Debug(Messages.GettingAllCommitsBetween, streamId, minRevision, maxRevision);
return(TryMongo(() =>
{
IMongoQuery query = Query.And(Query.EQ("_id.StreamId", streamId),
Query.GTE("Events.StreamRevision", minRevision),
Query.LTE("Events.StreamRevision", maxRevision));
return PersistedCommits.Find(query).SetSortOrder("Events.StreamRevision").Select(mc => mc.ToCommit(_serializer));
}));
}
private ICommit PersistWithClientSideOptimisticLoop(CommitAttempt attempt)
{
return(TryMongo(() =>
{
BsonDocument commitDoc = attempt.ToMongoCommit(
_getNextCheckpointNumber(),
_serializer
);
bool retry = true;
while (retry)
{
try
{
// for concurrency / duplicate commit detection safe mode is required
PersistedCommits.Insert(commitDoc, _insertCommitWriteConcern);
retry = false;
UpdateStreamHeadAsync(attempt.BucketId, attempt.StreamId, attempt.StreamRevision, attempt.Events.Count);
Logger.Debug(Messages.CommitPersisted, attempt.CommitId);
}
catch (MongoException e)
{
if (!e.Message.Contains(ConcurrencyException))
{
throw;
}
// checkpoint index?
if (e.Message.Contains(MongoCommitIndexes.CheckpointNumberMMApV1) ||
e.Message.Contains(MongoCommitIndexes.CheckpointNumberWiredTiger))
{
commitDoc[MongoCommitFields.CheckpointNumber] = _getNextCheckpointNumber().LongValue;
}
else
{
ICommit savedCommit = PersistedCommits.FindOne(attempt.ToMongoCommitIdQuery()).ToCommit(_serializer);
if (savedCommit.CommitId == attempt.CommitId)
{
throw new DuplicateCommitException();
}
Logger.Debug(Messages.ConcurrentWriteDetected);
throw new ConcurrencyException();
}
}
}
return commitDoc.ToCommit(_serializer);
}));
}
