public void SetUp(IProjection[] projections, int version, Boolean setupMetrics) { Checkpoint versionInfo = _checkpoints.FindOneById("VERSION") ?? new Checkpoint("VERSION", 0, null); int currentVersion = (Int32)versionInfo.Value; Int64 projectionStartFormCheckpointValue = 0L; if (currentVersion == 0) { var eventsVersion = _checkpoints.FindOneById(EventStoreFactory.PartitionCollectionName); if (eventsVersion != null) { projectionStartFormCheckpointValue = eventsVersion.Value; } } //set all projection to active = false _checkpoints.UpdateMany( Builders <Checkpoint> .Filter.Ne(c => c.Slot, null), Builders <Checkpoint> .Update.Set(c => c.Active, false) ); foreach (var projection in projections) { Add(projection, projectionStartFormCheckpointValue); } // mark db if (version > currentVersion) { versionInfo.Value = version; _checkpoints.Save(versionInfo, versionInfo.Id); } foreach (var slot in projections.Select(p => p.Info.SlotName).Distinct()) { var slotName = slot; if (setupMetrics) { KernelMetricsHelper.SetCheckpointCountToDispatch(slot, () => GetCheckpointCount(slotName)); } _checkpointSlotTracker[slot] = 0; _slotRebuildTracker[slot] = false; } if (setupMetrics) { KernelMetricsHelper.SetCheckpointCountToDispatch("", GetCheckpointMaxCount); } }
private void CreateTplChain(ref BufferBlock <AtomicDispatchChunk> buffer, ref ITargetBlock <AtomicDispatchChunk> broadcaster) { DataflowBlockOptions bufferOptions = new DataflowBlockOptions { BoundedCapacity = 4000, }; var localBuffer = buffer = new BufferBlock <AtomicDispatchChunk>(bufferOptions); Metric.Gauge("atomic-projection-buffer", () => localBuffer.Count, Unit.Items); ExecutionDataflowBlockOptions enhancerExecutionOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = 4000, MaxDegreeOfParallelism = 1, }; var enhancer = new TransformBlock <AtomicDispatchChunk, AtomicDispatchChunk>(c => { _commitEnhancer.Enhance(c.Chunk); return(c); }, enhancerExecutionOptions); buffer.LinkTo(enhancer, new DataflowLinkOptions() { PropagateCompletion = true }); Metric.Gauge("atomic-projection-enhancer-buffer", () => enhancer.InputCount, Unit.Items); ExecutionDataflowBlockOptions consumerExecutionOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = 15000, MaxDegreeOfParallelism = 1, }; var consumers = new List <ITargetBlock <AtomicDispatchChunk> >(); foreach (var item in _consumerBlocks) { //Ok I have a list of atomic projection, we need to create projector for every item. var blocks = item.Value; var consumer = new ActionBlock <AtomicDispatchChunk>(InnerDispatch(item, blocks), consumerExecutionOptions); Metric.Gauge("atomic-projection-consumer-buffer-" + item.Key.Name, () => consumer.InputCount, Unit.Items); KernelMetricsHelper.CreateMeterForAtomicReadmodelDispatcherCount(item.Key.Name); consumers.Add(consumer); } broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, "AtomicPoller", 3000); enhancer.LinkTo(broadcaster, new DataflowLinkOptions() { PropagateCompletion = true }); }
private async Task InitAsync() { _maxDispatchedCheckpoint = 0; DumpProjections(); TenantContext.Enter(_config.TenantId); await _housekeeper.InitAsync().ConfigureAwait(false); await ConfigureProjectionsAsync().ConfigureAwait(false); // cleanup await _housekeeper.RemoveAllAsync(_persistence).ConfigureAwait(false); var allSlots = _projectionsBySlot.Keys.ToArray(); var allClients = new List <ICommitPollingClient>(); //recreate all polling clients. foreach (var bucket in _config.BucketInfo) { string pollerId = "bucket: " + String.Join(",", bucket.Slots); var client = _pollingClientFactory.Create(_persistence, pollerId); allClients.Add(client); _bucketToClient.Add(bucket, client); client.Configure(GetStartGlobalCheckpoint(bucket.Slots), 4000); } _clients = allClients.ToArray(); foreach (var slotName in allSlots) { KernelMetricsHelper.CreateMeterForDispatcherCountSlot(slotName); var startCheckpoint = GetStartCheckpointForSlot(slotName); _logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint); var name = slotName; //find right consumer var slotBucket = _config.BucketInfo.SingleOrDefault(b => b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ?? _config.BucketInfo.Single(b => b.Slots[0] == "*"); var client = _bucketToClient[slotBucket]; client.AddConsumer($"SLOT: {slotName}", commit => DispatchCommitAsync(commit, name, startCheckpoint)); } Initialized = true; KernelMetricsHelper.SetProjectionEngineCurrentDispatchCount(() => _countOfConcurrentDispatchingCommit); }
public void InitSut() { var config = new ProjectionEngineConfig(); projections = new [] { new Projection(Substitute.For <ICollectionWrapper <SampleReadModel, String> >()) }; sut = new RebuildProjectionSlotDispatcher( NullLogger.Instance, slotName, config, projections, 4, NullLoggerThreadContextManager.Instance); //Needed to avoid crash on wrong metrics dispatch. KernelMetricsHelper.CreateMeterForRebuildDispatcherBuffer(slotName, () => 0); }
private void RegisterHealthChecks(string pollerName) { KernelMetricsHelper.SetCommitPollingClientBufferSize(pollerName, () => GetClientBufferSize()); //Set health check for polling HealthChecks.RegisterHealthCheck("Polling-" + pollerName, () => { if (Status == CommitPollingClientStatus.Stopped) { //poller is stopped, system healty return(HealthCheckResult.Healthy("Automatic polling stopped")); } else if (Status == CommitPollingClientStatus.Faulted || LastException != null) { //poller is stopped, system is not healty anymore. var exceptionText = (LastException != null ? LastException.ToString() : ""); exceptionText = exceptionText.Replace("{", "{{").Replace("}", "}}"); return(HealthCheckResult.Unhealthy( "[LastDispatchedPosition: {0}] - Faulted (exception in consumer): {1}", _lastDispatchedPosition, exceptionText)); } return(HealthCheckResult.Healthy("Poller alive")); }); //Now register health check for the internal NES poller, to diagnose errors in polling. HealthChecks.RegisterHealthCheck("Polling internal errors: ", () => { if (_innerSubscription?.LastException != null) { return(HealthCheckResult.Unhealthy("[LastDispatchedPosition: {0}] - Inner NStore poller has error: {1}", _lastDispatchedPosition, _innerSubscription?.LastException)); } return(HealthCheckResult.Healthy("Inner NES Poller Ok")); }); }
private Func <AtomicDispatchChunk, Task> InnerDispatch( KeyValuePair <Type, List <AtomicDispatchChunkConsumer> > item, List <AtomicDispatchChunkConsumer> blocks) { return(async dispatchObj => { try { if (dispatchObj.Chunk.Payload is Changeset changeset) { var aggregateId = changeset.GetIdentity(); if (aggregateId?.GetType() == item.Key) { //Remember to dispatch this only to objects that are associated to this poller. Remember that we build a single //TPL chain, so both the poller (default and catchup) will push data inside the very same tpl pipeline, but inside //this dispatcher chain, we simply dispatch only to the blocks that have correct poller id. QueryPerformanceCounter(out long ticks1); foreach (var atomicDispatchChunkConsumer in blocks.Where(_ => _.PollerId == dispatchObj.PollerId)) { if (Logger.IsDebugEnabled) { Logger.DebugFormat("Dispatched chunk {0} with poller {1} for readmodel {2}", dispatchObj.Chunk.Position, dispatchObj.PollerId, atomicDispatchChunkConsumer.Consumer.AtomicReadmodelInfoAttribute.Name); } //let the abstract readmodel handle everyting, then finally dispatch notification.Handle method internally //catches exception and make atomic readmodel faulty. var handleReturnValue = await atomicDispatchChunkConsumer.Consumer.Handle( dispatchObj.Chunk.Position, changeset, aggregateId).ConfigureAwait(false); if (handleReturnValue != null) { #pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed //I do not care if the notification works or not, I simply call it and forget. if (handleReturnValue.CreatedForFirstTime) { AtomicReadmodelNotifier.ReadmodelCreatedAsync(handleReturnValue.Readmodel, changeset); } else { AtomicReadmodelNotifier.ReadmodelUpdatedAsync(handleReturnValue.Readmodel, changeset); } #pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed } } QueryPerformanceCounter(out long ticks2); var elapsedTicks = ticks2 - ticks1; KernelMetricsHelper.IncrementProjectionCounterAtomicProjection(item.Key.Name, elapsedTicks); KernelMetricsHelper.MarkCommitDispatchedAtomicReadmodelCount(item.Key.Name, 1); } } foreach (var rm in blocks.Where(_ => _.PollerId == dispatchObj.PollerId)) { _atomicProjectionCheckpointManager.MarkPosition(rm.Consumer.AtomicReadmodelInfoAttribute.Name, dispatchObj.Chunk.Position); } } catch (Exception ex) { //TODO: Implement health check with failed _engineExceptions[item.Key.Name] = ex; Logger.ErrorFormat(ex, "Generic error in TPL dispatching {0} with poller {1} for id type {2}", dispatchObj.Chunk.Position, dispatchObj.PollerId, item.Key); throw; } }); }
internal async Task DispatchEventAsync(UnwindedDomainEvent unwindedEvent) { if (unwindedEvent == UnwindedDomainEvent.LastEvent) { Finished = true; _lastCheckpointRebuilded = LastCheckpointDispatched; //Set to zero metrics, we dispatched everything. return; } var chkpoint = unwindedEvent.CheckpointToken; if (chkpoint > LastCheckpointDispatched) { if (_logger.IsDebugEnabled) { _logger.DebugFormat("Discharded event {0} commit {1} because last checkpoint dispatched for slot {2} is {3}.", unwindedEvent.CommitId, unwindedEvent.CheckpointToken, SlotName, _maxCheckpointDispatched); } return; } Interlocked.Increment(ref RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit); TenantContext.Enter(_config.TenantId); try { string eventName = unwindedEvent.EventType; foreach (var projection in _projections) { var cname = projection.Info.CommonName; long elapsedticks; try { QueryPerformanceCounter(out long ticks1); await projection.HandleAsync(unwindedEvent.GetEvent(), true).ConfigureAwait(false); QueryPerformanceCounter(out long ticks2); elapsedticks = ticks2 - ticks1; KernelMetricsHelper.IncrementProjectionCounterRebuild(cname, SlotName, eventName, elapsedticks); } catch (Exception ex) { _logger.FatalFormat(ex, "[Slot: {3} Projection: {4}] Failed checkpoint: {0} StreamId: {1} Event Name: {2}", unwindedEvent.CheckpointToken, unwindedEvent.PartitionId, eventName, SlotName, cname ); HealthChecks.RegisterHealthCheck($"RebuildDispatcher, slot {SlotName} - FailedCheckpoint {unwindedEvent.CheckpointToken}", () => HealthCheckResult.Unhealthy(ex) ); throw; } _metrics.Inc(cname, eventName, elapsedticks); if (_logger.IsDebugEnabled) { _logger.DebugFormat("[{3}] [{4}] Handled checkpoint {0}: {1} > {2}", unwindedEvent.CheckpointToken, unwindedEvent.PartitionId, eventName, SlotName, cname ); } } } catch (Exception ex) { _logger.ErrorFormat(ex, "Error dispathing commit id: {0}\nMessage: {1}\nError: {2}", unwindedEvent.CheckpointToken, unwindedEvent.Event, ex.Message); HealthChecks.RegisterHealthCheck($"RebuildDispatcher, slot {SlotName} - GeneralError", () => HealthCheckResult.Unhealthy(ex) ); throw; } _lastCheckpointRebuilded = chkpoint; KernelMetricsHelper.MarkEventInRebuildDispatchedCount(SlotName, 1); Interlocked.Decrement(ref RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit); }
/// <summary> /// Start rebuild process. /// </summary> /// <returns></returns> public async Task <RebuildStatus> RebuildAsync() { if (Logger.IsInfoEnabled) { Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId); } DumpProjections(); await _eventUnwinder.UnwindAsync().ConfigureAwait(false); _status = new RebuildStatus(); TenantContext.Enter(_config.TenantId); await ConfigureProjections().ConfigureAwait(false); var allSlots = _projectionsBySlot.Keys.ToArray(); //initialize dispatching of the commits foreach (var bucket in _config.BucketInfo) { _consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>()); _status.AddBucket(); } //Setup the slots foreach (var slotName in allSlots) { Logger.InfoFormat("Slot {0} will be rebuilded", slotName); var projectionsForThisSlot = _projectionsBySlot[slotName]; Int64 maximumDispatchedValue = projectionsForThisSlot .Select(p => _checkpointTracker.GetCheckpoint(p)) .Max(); var dispatcher = new RebuildProjectionSlotDispatcher(Logger, slotName, _config, projectionsForThisSlot, maximumDispatchedValue, _loggerThreadContextManager); KernelMetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch); _rebuildDispatchers.Add(dispatcher); //find right consumer var slotBucket = _config.BucketInfo.SingleOrDefault(b => b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ?? _config.BucketInfo.Single(b => b.Slots[0] == "*"); var consumerList = _consumers[slotBucket]; consumerList.Add(dispatcher); } //Creates TPL chain and start polling everything on the consumer foreach (var consumer in _consumers) { var bucketInfo = String.Join(",", consumer.Key.Slots); if (consumer.Value.Count == 0) { Logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo); _status.BucketDone(bucketInfo, 0, 0, 0); continue; } var consumerBufferOptions = new DataflowBlockOptions(); consumerBufferOptions.BoundedCapacity = consumer.Key.BufferSize; var _buffer = new BufferBlock <UnwindedDomainEvent>(consumerBufferOptions); ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions(); executionOption.BoundedCapacity = consumer.Key.BufferSize; var dispatcherList = consumer.Value; _projectionInspector.ResetHandledEvents(); List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> > consumers = new List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> >(); foreach (var dispatcher in dispatcherList) { ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions(); consumerOptions.BoundedCapacity = consumer.Key.BufferSize; var actionBlock = new ActionBlock <UnwindedDomainEvent>((Func <UnwindedDomainEvent, Task>)dispatcher.DispatchEventAsync, consumerOptions); HashSet <Type> eventsOfThisSlot = new HashSet <Type>(); foreach (var projection in dispatcher.Projections) { var domainEventTypesHandledByThisProjection = _projectionInspector.InspectProjectionForEvents(projection.GetType()); foreach (var type in domainEventTypesHandledByThisProjection) { eventsOfThisSlot.Add(type); } } SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> slotInfo = new SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent>( actionBlock, dispatcher.SlotName, eventsOfThisSlot); consumers.Add(slotInfo); KernelMetricsHelper.CreateMeterForRebuildDispatcherBuffer(dispatcher.SlotName, () => actionBlock.InputCount); } var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList(); var broadcaster = SlotGuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, consumer.Key.BufferSize); _buffer.LinkTo(broadcaster, new DataflowLinkOptions() { PropagateCompletion = true }); KernelMetricsHelper.CreateGaugeForRebuildFirstBuffer(bucketInfo, () => _buffer.Count); KernelMetricsHelper.CreateGaugeForRebuildBucketDBroadcasterBuffer(bucketInfo, () => broadcaster.InputCount); KernelMetricsHelper.CreateMeterForRebuildEventCompleted(bucketInfo); //fire each bucket in own thread #pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed Task.Factory.StartNew(() => StartPoll( _buffer, broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers)); //await StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers).ConfigureAwait(false); #pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed } KernelMetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit); return(_status); }
private async Task DispatchCommitAsync(IChunk chunk, string slotName, Int64 startCheckpoint) { Interlocked.Increment(ref _countOfConcurrentDispatchingCommit); _loggerThreadContextManager.SetContextProperty("commit", $"{chunk.OperationId}/{chunk.Position}"); if (_logger.IsDebugEnabled) { _logger.DebugFormat("Dispatching checkpoit {0} on tenant {1}", chunk.Position, _config.TenantId); } TenantContext.Enter(_config.TenantId); var chkpoint = chunk.Position; if (!lastCheckpointDispatched.ContainsKey(slotName)) { lastCheckpointDispatched[slotName] = 0; } if (lastCheckpointDispatched[slotName] >= chkpoint) { var error = String.Format("Sequence broken, last checkpoint for slot {0} was {1} and now we dispatched {2}", slotName, lastCheckpointDispatched[slotName], chkpoint); _logger.Error(error); throw new JarvisFrameworkEngineException(error); } if (lastCheckpointDispatched[slotName] + 1 != chkpoint && lastCheckpointDispatched[slotName] > 0) { _logger.DebugFormat("Sequence of commit not consecutive (probably holes), last dispatched {0} receiving {1}", lastCheckpointDispatched[slotName], chkpoint); } lastCheckpointDispatched[slotName] = chkpoint; if (chkpoint <= startCheckpoint) { //Already dispatched, skip it. Interlocked.Decrement(ref _countOfConcurrentDispatchingCommit); return; } var projections = _projectionsBySlot[slotName]; object[] events = GetArrayOfObjectFromChunk(chunk); var eventCount = events.Length; //now it is time to dispatch the commit, we will dispatch each projection //and for each projection we dispatch all the events present in changeset bool someProjectionProcessedTheEvent = false; foreach (var projection in projections) { var cname = projection.Info.CommonName; Object eventMessage = null; try { //Foreach projection we need to dispach every event for (int index = 0; index < eventCount; index++) { eventMessage = events[index]; var message = eventMessage as IMessage; string eventName = eventMessage.GetType().Name; _loggerThreadContextManager.SetContextProperty("evType", eventName); _loggerThreadContextManager.SetContextProperty("evMsId", message?.MessageId); _loggerThreadContextManager.SetContextProperty("evCheckpointToken", chunk.Position); _loggerThreadContextManager.SetContextProperty("prj", cname); var checkpointStatus = _checkpointTracker.GetCheckpointStatus(cname, chunk.Position); long ticks = 0; try { //pay attention, stopwatch consumes time. var sw = new Stopwatch(); sw.Start(); var eventProcessed = await projection .HandleAsync(eventMessage, checkpointStatus.IsRebuilding) .ConfigureAwait(false); someProjectionProcessedTheEvent |= eventProcessed; sw.Stop(); ticks = sw.ElapsedTicks; KernelMetricsHelper.IncrementProjectionCounter(cname, slotName, eventName, ticks, sw.ElapsedMilliseconds); if (_logger.IsDebugEnabled) { _logger.DebugFormat("[Slot:{3}] [Projection {4}] Handled checkpoint {0}: {1} > {2}", chunk.Position, chunk.PartitionId, $"eventName: {eventName} [event N°{index}]", slotName, cname ); } } catch (Exception ex) { var error = String.Format("[Slot: {3} Projection: {4}] Failed checkpoint: {0} StreamId: {1} Event Name: {2}", chunk.Position, chunk.PartitionId, eventName, slotName, cname); _logger.Fatal(error, ex); _engineFatalErrors.Add(String.Format("{0}\n{1}", error, ex)); throw; } } //End of event cycle } catch (Exception ex) { _loggerThreadContextManager.ClearContextProperty("commit"); _logger.ErrorFormat(ex, "Error dispathing Chunk [{0}]\n Message: {1}\n Error: {2}\n", chunk.Position, eventMessage?.GetType()?.Name, ex.Message); throw; } finally { _loggerThreadContextManager.ClearContextProperty("evType"); _loggerThreadContextManager.ClearContextProperty("evMsId"); _loggerThreadContextManager.ClearContextProperty("evCheckpointToken"); _loggerThreadContextManager.ClearContextProperty("prj"); } projection.CheckpointProjected(chunk.Position); //TODO: Evaluate if it is needed to update single projection checkpoint //Update this projection, set all events of this checkpoint as dispatched. //await _checkpointTracker.UpdateProjectionCheckpointAsync(cname, chunk.Position).ConfigureAwait(false); } //End of projection cycle. await _checkpointTracker.UpdateSlotAndSetCheckpointAsync( slotName, projections.Select(_ => _.Info.CommonName), chunk.Position, someEventDispatched : someProjectionProcessedTheEvent).ConfigureAwait(false); KernelMetricsHelper.MarkCommitDispatchedCount(slotName, 1); await _notifyCommitHandled.SetDispatched(slotName, chunk).ConfigureAwait(false); // ok in multithread wihout locks! if (_maxDispatchedCheckpoint < chkpoint) { if (_logger.IsDebugEnabled) { _logger.DebugFormat("Updating last dispatched checkpoint from {0} to {1}", _maxDispatchedCheckpoint, chkpoint ); } _maxDispatchedCheckpoint = chkpoint; } _loggerThreadContextManager.ClearContextProperty("commit"); Interlocked.Decrement(ref _countOfConcurrentDispatchingCommit); }