public virtual IEnumerable <ICommit> GetFrom(string checkpointToken) { Logger.Debug(Messages.GettingAllCommitsFromCheckpoint, checkpointToken); LongCheckpoint checkpoint = LongCheckpoint.Parse(checkpointToken); return(QueryCommits <RavenCommitByCheckpoint>(x => x.CheckpointNumber > checkpoint.LongValue, x => x.CheckpointNumber)); }
public void RebuildStarted(IProjection projection, String lastCommitId) { var projectionName = projection.GetCommonName(); _checkpoints.Update( Query.EQ("_id", projectionName), Update <Checkpoint> .Set(x => x.RebuildStart, DateTime.UtcNow) .Set(x => x.RebuildStop, null) .Set(x => x.RebuildTotalSeconds, 0) .Set(x => x.RebuildActualSeconds, 0) .Set(x => x.Current, null) .Set(x => x.Events, 0) .Set(x => x.Details, null) ); //Check if some commit was deleted and lastCommitId in Eventstore //is lower than the latest dispatched commit. var trackerLastValue = LongCheckpoint.Parse(_checkpointTracker[projection.GetCommonName()]).LongValue; var lastCommitIdLong = LongCheckpoint.Parse(lastCommitId).LongValue; if (trackerLastValue > 0) { //new projection, it has no dispatched checkpoint. _slotRebuildTracker[_projectionToSlot[projectionName]] = true; } if (lastCommitIdLong < trackerLastValue) { _checkpointTracker[projection.GetCommonName()] = lastCommitId; _checkpoints.Update( Query.EQ("_id", projectionName), Update <Checkpoint> .Set(x => x.Value, lastCommitId)); } _higherCheckpointToDispatchInRebuild = Math.Max( _higherCheckpointToDispatchInRebuild, Math.Min(trackerLastValue, lastCommitIdLong)); }
void ConfigureProjections() { _checkpointTracker.SetUp(_allProjections, 1); var lastCommit = GetLastCommitId(); foreach (var slot in _projectionsBySlot) { var maxCheckpointDispatchedInSlot = slot.Value .Select(projection => { var checkpoint = _checkpointTracker.GetCheckpoint(projection); var longCheckpoint = LongCheckpoint.Parse(checkpoint); return(longCheckpoint.LongValue); }) .Max(); foreach (var projection in slot.Value) { //Rebuild the slot only if it has at least one dispatched commit if (_checkpointTracker.NeedsRebuild(projection) && maxCheckpointDispatchedInSlot >= 0) { //be sure that projection in rebuild has the same Last dispatched commit fo the entire slot _checkpointTracker.SetCheckpoint(projection.GetCommonName(), maxCheckpointDispatchedInSlot.ToString()); projection.Drop(); projection.StartRebuild(_rebuildContext); _checkpointTracker.RebuildStarted(projection, lastCommit); } projection.SetUp(); } } }
public IEnumerable <ICommit> GetFrom(string bucketId, string checkpointToken) { var intCheckpoint = LongCheckpoint.Parse(checkpointToken); Logger.Debug(Messages.GettingAllCommitsFromBucketAndCheckpoint, bucketId, intCheckpoint.Value); return(QueryCommits <RavenCommitByCheckpoint>(x => x.BucketId == bucketId && x.CheckpointNumber > intCheckpoint.LongValue, x => x.CheckpointNumber)); }
string GetStartCheckpointForSlot(string slotName) { LongCheckpoint min = null; var projections = _projectionsBySlot[slotName]; foreach (var projection in projections) { if (_checkpointTracker.NeedsRebuild(projection)) { return(null); } var currentValue = _checkpointTracker.GetCurrent(projection); if (currentValue == null) { return(null); } var currentCheckpoint = LongCheckpoint.Parse(currentValue); if (min == null || currentCheckpoint.LongValue < min.LongValue) { min = currentCheckpoint; } } return(min.Value); }
string GetStartGlobalCheckpoint() { var checkpoints = _projectionsBySlot.Keys.Select(GetStartCheckpointForSlot).Distinct().ToArray(); var min = checkpoints.Any() ? checkpoints.Min(x => LongCheckpoint.Parse(x).LongValue) : 0; return(new LongCheckpoint(min).Value); }
public void should_have_a_checkpoint_greater_than_the_previous_commit_on_the_other_process() { long chkNum1 = LongCheckpoint.Parse(_commit1.CheckpointToken).LongValue; long chkNum2 = LongCheckpoint.Parse(_commit2.CheckpointToken).LongValue; chkNum2.ShouldBeGreaterThan(chkNum1); }
public static BsonDocument ToMongoCommit(this CommitAttempt commit, LongCheckpoint checkpoint, IDocumentSerializer serializer) { int streamRevision = commit.StreamRevision - (commit.Events.Count - 1); int streamRevisionStart = streamRevision; IEnumerable <BsonDocument> events = commit .Events .Select(e => new BsonDocument { { MongoCommitFields.StreamRevision, streamRevision++ }, { MongoCommitFields.Payload, new BsonDocumentWrapper(typeof(EventMessage), serializer.Serialize(e)) } }); return(new BsonDocument { { MongoCommitFields.CheckpointNumber, checkpoint.LongValue }, { MongoCommitFields.CommitId, commit.CommitId }, { MongoCommitFields.CommitStamp, commit.CommitStamp }, { MongoCommitFields.Headers, BsonDocumentWrapper.Create(commit.Headers) }, { MongoCommitFields.Events, new BsonArray(events) }, { MongoCommitFields.Dispatched, false }, { MongoCommitFields.StreamRevisionFrom, streamRevisionStart }, { MongoCommitFields.StreamRevisionTo, streamRevision - 1 }, { MongoCommitFields.BucketId, commit.BucketId }, { MongoCommitFields.StreamId, commit.StreamId }, { MongoCommitFields.CommitSequence, commit.CommitSequence } }); }
protected override void Context() { var commit = Persistence.Commit(Guid.NewGuid().ToString().BuildAttempt()); _checkpointBeforePurge = LongCheckpoint.Parse(commit.CheckpointToken); Persistence.DeleteStream(commit.StreamId); Persistence.Purge("default"); }
public Task <ICheckpoint> GetCheckpoint(string checkpointToken) { if (string.IsNullOrWhiteSpace(checkpointToken)) { return(Task.FromResult <ICheckpoint>(new LongCheckpoint(-1))); } return(Task.FromResult <ICheckpoint>(LongCheckpoint.Parse(checkpointToken))); }
public ICheckpoint GetCheckpoint(string checkpointToken) { if (string.IsNullOrWhiteSpace(checkpointToken)) { return(new LongCheckpoint(-1)); } return(LongCheckpoint.Parse(checkpointToken)); }
public void OnNext(ICommit value) { var chkpoint = LongCheckpoint.Parse(value.CheckpointToken).LongValue; if (chkpoint > _lastCheckpoint) { _counter++; } _lastCheckpoint = chkpoint; }
void ConfigureProjections() { _checkpointTracker.SetUp(_allProjections, 1); var lastCommit = GetLastCommitId(); foreach (var slot in _projectionsBySlot) { var maxCheckpointDispatchedInSlot = slot.Value .Select(projection => { var checkpoint = _checkpointTracker.GetCheckpoint(projection); var longCheckpoint = LongCheckpoint.Parse(checkpoint); return(longCheckpoint.LongValue); }) .Max(); foreach (var projection in slot.Value) { if (_checkpointTracker.NeedsRebuild(projection)) { //Check if this slot has ever dispatched at least one commit if (maxCheckpointDispatchedInSlot > 0) { _checkpointTracker.SetCheckpoint(projection.GetCommonName(), maxCheckpointDispatchedInSlot.ToString()); projection.Drop(); projection.StartRebuild(_rebuildContext); _checkpointTracker.RebuildStarted(projection, lastCommit); } else { //this is a new slot, all the projection should execute //as if they are not in rebuild, because they are new //so we need to immediately stop rebuilding. projection.StopRebuild(); } } projection.SetUp(); } } var errors = _checkpointTracker.GetCheckpointErrors(); if (errors.Any()) { StringBuilder fullError = new StringBuilder(); foreach (var error in errors) { Logger.ErrorFormat("CheckpointError: {0}", error); fullError.AppendLine(error); } throw new Exception(String.Format("Found {0} errors in checkpoint status: {1}", errors.Count, fullError.ToString())); } }
public IEnumerable <ICommit> GetFrom(string checkpointToken) { Logger.Debug(Resources.GettingAllCommitsFromCheckpoint, checkpointToken); ICheckpoint checkpoint = LongCheckpoint.Parse(checkpointToken); return(_buckets .Values .SelectMany(b => b.GetCommits()) .Where(c => c.Checkpoint.CompareTo(checkpoint) > 0) .OrderBy(c => c.Checkpoint) .ToArray()); }
public CheckPointReplayStatus GetCheckpointStatus(string id, string checkpoint) { string lastDispatched = _checkpointTracker[id]; var currentCheckpointValue = LongCheckpoint.Parse(checkpoint).LongValue; long lastDispatchedValue = LongCheckpoint.Parse(lastDispatched).LongValue; bool isLast = currentCheckpointValue == lastDispatchedValue; bool isReplay = currentCheckpointValue <= lastDispatchedValue; _checkpointSlotTracker[_projectionToSlot[id]] = _higherCheckpointToDispatchInRebuild - currentCheckpointValue; return(new CheckPointReplayStatus(isLast, isReplay)); }
public IEnumerable <ICommit> GetFrom(string checkpointToken) { LongCheckpoint checkpoint = LongCheckpoint.Parse(checkpointToken); Logger.Debug(Messages.GettingAllCommitsFromCheckpoint, checkpointToken); return(ExecuteQuery(query => { string statement = _dialect.GetCommitsFromCheckpoint; query.AddParameter(_dialect.CheckpointNumber, checkpoint.LongValue); return query.ExecutePagedQuery(statement, (q, r) => { }) .Select(x => x.GetCommit(_serializer, _dialect)); })); }
public void SetUp(IProjection[] projections, int version) { var versionInfo = _checkpoints.FindOneById("VERSION"); if (versionInfo == null) { versionInfo = new Checkpoint("VERSION", "0"); } int currentVersion = int.Parse(versionInfo.Value); string projectionStartFormCheckpointValue = new LongCheckpoint(0).Value; if (currentVersion == 0) { var eventsVersion = _checkpoints.FindOneById("events"); if (eventsVersion != null) { projectionStartFormCheckpointValue = eventsVersion.Value; } } //set all projection to active = false _checkpoints.Update( Query <Checkpoint> .NE(c => c.Slot, null), Update <Checkpoint> .Set(c => c.Active, false), UpdateFlags.Multi ); foreach (var projection in projections) { Add(projection, projectionStartFormCheckpointValue); } // mark db if (version > currentVersion) { versionInfo.Value = new LongCheckpoint(version).Value; _checkpoints.Save(versionInfo); } foreach (var slot in projections.Select(p => p.GetSlotName()).Distinct()) { var slotName = slot; MetricsHelper.SetCheckpointCountToDispatch(slot, () => GetCheckpointCount(slotName)); _checkpointSlotTracker[slot] = 0; _slotRebuildTracker[slot] = false; } MetricsHelper.SetCheckpointCountToDispatch("", GetCheckpointMaxCount); }
public IEnumerable <ICommit> GetFrom(string bucketId, string checkpointToken) { LongCheckpoint checkpoint = LongCheckpoint.Parse(checkpointToken); Logger.Debug(Messages.GettingAllCommitsFromBucketAndCheckpoint, bucketId, checkpointToken); return(this.ExecuteQuery(query => { string statement = this._dialect.GetCommitsFromBucketAndCheckpoint; query.AddParameter(this._dialect.BucketId, bucketId, DbType.AnsiString); query.AddParameter(this._dialect.CheckpointNumber, checkpoint.LongValue); return query.ExecutePagedQuery(statement, (q, r) => { }) .Select(x => x.GetCommit(this._serializer, this._dialect)); })); }
private void Init() { _maxDispatchedCheckpoint = 0; DumpProjections(); TenantContext.Enter(_config.TenantId); _housekeeper.Init(); _eventstore = Wireup .Init() .LogTo(t => new NEventStoreLog4NetLogger(LoggerFactory.Create(t))) .UsingMongoPersistence(() => _config.EventStoreConnectionString, new DocumentObjectSerializer()) .InitializeStorageEngine() .Build(); ConfigureProjections(); // cleanup _housekeeper.RemoveAll(_eventstore.Advanced); var allSlots = _projectionsBySlot.Keys.ToArray(); //recreate all polling clients. foreach (var bucket in _config.BucketInfo) { var client = _pollingClientFactory(_eventstore.Advanced); _clients.Add(client); _bucketToClient.Add(bucket, client); } foreach (var slotName in allSlots) { MetricsHelper.CreateMeterForDispatcherCountSlot(slotName); var startCheckpoint = GetStartCheckpointForSlot(slotName); Logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint); var name = slotName; //find right consumer var slotBucket = _config.BucketInfo.SingleOrDefault(b => b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ?? _config.BucketInfo.Single(b => b.Slots[0] == "*"); var client = _bucketToClient[slotBucket]; client.AddConsumer(commit => DispatchCommit(commit, name, LongCheckpoint.Parse(startCheckpoint))); } MetricsHelper.SetProjectionEngineCurrentDispatchCount(() => _countOfConcurrentDispatchingCommit); }
public IEnumerable <ICommit> GetFrom(string checkpointToken) { var intCheckpoint = LongCheckpoint.Parse(checkpointToken); Logger.Debug(Messages.GettingAllCommitsFromCheckpoint, intCheckpoint.Value); return(TryMongo(() => PersistedCommits .Find( Query.And( Query.NE(MongoCommitFields.BucketId, MongoSystemBuckets.RecycleBin), Query.GT(MongoCommitFields.CheckpointNumber, intCheckpoint.LongValue) ) ) .SetSortOrder(MongoCommitFields.CheckpointNumber) .Select(x => x.ToCommit(_serializer)) )); }
private void Init() { _maxDispatchedCheckpoint = 0; DumpProjections(); _stopping = new ManualResetEventSlim(false); TenantContext.Enter(_config.TenantId); _housekeeper.Init(); _eventstore = Wireup .Init() .LogTo(t => new NEventStoreLog4NetLogger(LoggerFactory.Create(t))) .UsingMongoPersistence(() => _config.EventStoreConnectionString, new DocumentObjectSerializer()) .InitializeStorageEngine() .Build(); ConfigureProjections(); // cleanup _housekeeper.RemoveAll(_eventstore.Advanced); var allSlots = _projectionsBySlot.Keys.ToArray(); _client.Create(_eventstore.Advanced, _config.PollingMsInterval); var subscriptions = new List <IDisposable>(); _observeCommits = _client.ObserveFrom(GetStartGlobalCheckpoint()); foreach (var slotName in allSlots) { MetricsHelper.CreateMeterForDispatcherCountSlot(slotName); var startCheckpoint = GetStartCheckpointForSlot(slotName); Logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint); var name = slotName; subscriptions.Add( _observeCommits.Subscribe(commit => DispatchCommit(commit, name, LongCheckpoint.Parse(startCheckpoint)), HandleError)); } _subscriptions = subscriptions.ToArray(); }
public MongoPersistenceEngine(MongoDatabase store, IDocumentSerializer serializer, MongoPersistenceOptions options) { if (store == null) { throw new ArgumentNullException("store"); } if (serializer == null) { throw new ArgumentNullException("serializer"); } if (options == null) { throw new ArgumentNullException("options"); } _store = store; _serializer = serializer; _options = options; // set config options _commitSettings = _options.GetCommitSettings(); _snapshotSettings = _options.GetSnapshotSettings(); _streamSettings = _options.GetStreamSettings(); _insertCommitWriteConcern = _options.GetInsertCommitWriteConcern(); _getLastCheckPointNumber = () => TryMongo(() => { var max = PersistedCommits .FindAll() .SetFields(Fields.Include(MongoCommitFields.CheckpointNumber)) .SetSortOrder(SortBy.Descending(MongoCommitFields.CheckpointNumber)) .SetLimit(1) .FirstOrDefault(); return(max != null ? max[MongoCommitFields.CheckpointNumber].AsInt64 : 0L); }); _getNextCheckpointNumber = () => new LongCheckpoint(_getLastCheckPointNumber() + 1L); _updateScript = new BsonJavaScript("function (x){ return insertCommit(x);}"); _checkpointZero = new LongCheckpoint(0); }
public ICheckpoint ParseCheckpoint(string checkpointValue) { return(LongCheckpoint.Parse(checkpointValue)); }
public void should_have_checkpoint_equal_to_two() { LongCheckpoint.Parse(_commit.CheckpointToken).LongValue.ShouldBe(2); }
public ICheckpoint GetCheckpoint(string checkpointToken = null) { return(LongCheckpoint.Parse(checkpointToken)); }
public void should_have_correct_checkpoint() { LongCheckpoint.Parse(_checkpointToken).LongValue.ShouldBe(Clients * Iterations + 1); }
public void last_deleted_commit_has_the_higher_checkpoint_number() { LongCheckpoint.Parse(_commits[0].CheckpointToken).LongValue.ShouldBe(4); }
protected override void Because() { var commit = Persistence.Commit(Guid.NewGuid().ToString().BuildAttempt()); _checkpointAfterPurge = LongCheckpoint.Parse(commit.CheckpointToken); }
public Task <ICheckpoint> GetCheckpoint(string checkpointToken = null) { return(Task.FromResult <ICheckpoint>(LongCheckpoint.Parse(checkpointToken))); }
public ICheckpoint GetCheckpoint(string checkpointToken) { return(string.IsNullOrWhiteSpace(checkpointToken) ? null : LongCheckpoint.Parse(checkpointToken)); }