private void CreateTplChain(ref BufferBlock <AtomicDispatchChunk> buffer, ref ITargetBlock <AtomicDispatchChunk> broadcaster) { DataflowBlockOptions bufferOptions = new DataflowBlockOptions { BoundedCapacity = 4000, }; var localBuffer = buffer = new BufferBlock <AtomicDispatchChunk>(bufferOptions); Metric.Gauge("atomic-projection-buffer", () => localBuffer.Count, Unit.Items); ExecutionDataflowBlockOptions enhancerExecutionOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = 4000, MaxDegreeOfParallelism = 1, }; var enhancer = new TransformBlock <AtomicDispatchChunk, AtomicDispatchChunk>(c => { _commitEnhancer.Enhance(c.Chunk); return(c); }, enhancerExecutionOptions); buffer.LinkTo(enhancer, new DataflowLinkOptions() { PropagateCompletion = true }); Metric.Gauge("atomic-projection-enhancer-buffer", () => enhancer.InputCount, Unit.Items); ExecutionDataflowBlockOptions consumerExecutionOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = 15000, MaxDegreeOfParallelism = 1, }; var consumers = new List <ITargetBlock <AtomicDispatchChunk> >(); foreach (var item in _consumerBlocks) { //Ok I have a list of atomic projection, we need to create projector for every item. var blocks = item.Value; var consumer = new ActionBlock <AtomicDispatchChunk>(InnerDispatch(item, blocks), consumerExecutionOptions); Metric.Gauge("atomic-projection-consumer-buffer-" + item.Key.Name, () => consumer.InputCount, Unit.Items); KernelMetricsHelper.CreateMeterForAtomicReadmodelDispatcherCount(item.Key.Name); consumers.Add(consumer); } broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, "AtomicPoller", 3000); enhancer.LinkTo(broadcaster, new DataflowLinkOptions() { PropagateCompletion = true }); }
private void CreateTplChain() { DataflowBlockOptions bufferOptions = new DataflowBlockOptions(); bufferOptions.BoundedCapacity = _bufferSize; _buffer = new BufferBlock <IChunk>(bufferOptions); ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions(); executionOption.BoundedCapacity = _bufferSize; _broadcaster = GuaranteedDeliveryBroadcastBlock.Create(_consumers, _id, 3000); _buffer.LinkTo(_broadcaster, new DataflowLinkOptions() { PropagateCompletion = true }); }
private void CreateTplChain(ref BufferBlock <AtomicDispatchChunk> buffer, ref ITargetBlock <AtomicDispatchChunk> broadcaster) { DataflowBlockOptions bufferOptions = new DataflowBlockOptions { BoundedCapacity = 4000, }; var localBuffer = buffer = new BufferBlock <AtomicDispatchChunk>(bufferOptions); Metrics.Metric.Gauge("atomic-projection-buffer", () => localBuffer.Count, Metrics.Unit.Items); ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions { BoundedCapacity = 4000, }; var enhancer = new TransformBlock <AtomicDispatchChunk, AtomicDispatchChunk>(c => { _commitEnhancer.Enhance(c.Chunk); //This was already done by the poller, but we are investigating on using directly a NStore poller. return(c); }, executionOption); buffer.LinkTo(enhancer, new DataflowLinkOptions() { PropagateCompletion = true }); Metrics.Metric.Gauge("atomic-projection-enhancer-buffer", () => (Double)enhancer.InputCount, Metrics.Unit.Items); var consumers = new List <ITargetBlock <AtomicDispatchChunk> >(); foreach (var item in _consumerBlocks) { //Ok I have a list of atomic projection, we need to create projector for every item. var blocks = item.Value; var consumer = new ActionBlock <AtomicDispatchChunk>(InnerDispatch(item, blocks), executionOption); Metrics.Metric.Gauge("atomic-projection-consumer-buffer-" + item.Key.Name, () => consumer.InputCount, Metrics.Unit.Items); consumers.Add(consumer); } broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, "AtomicPoller", 3000); enhancer.LinkTo(broadcaster, new DataflowLinkOptions() { PropagateCompletion = true }); }
public PollingObserveCommitsTpl( IPersistStreams persistStreams, int interval, string checkpointToken, ICommitEnhancer enhancer, bool boost) { _persistStreams = persistStreams; _checkpointToken = checkpointToken; _enhancer = enhancer; _interval = interval; _boost = boost; DataflowBlockOptions bufferOptions = new DataflowBlockOptions(); if (_boost) { bufferOptions.BoundedCapacity = 1000; } _buffer = new BufferBlock <ICommit>(bufferOptions); ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions(); executionOption.BoundedCapacity = 1000; executionOption.MaxDegreeOfParallelism = 16; _enhancerBlock = new TransformBlock <ICommit, ICommit>((Func <ICommit, ICommit>)Enhance, executionOption); _subscribers = new ConcurrentDictionary <Guid, ITargetBlock <ICommit> >(); _broadcaster = GuaranteedDeliveryBroadcastBlock.Create(_subscribers, 1000); _buffer.LinkTo(_enhancerBlock, new DataflowLinkOptions() { PropagateCompletion = true }); _enhancerBlock.LinkTo(_broadcaster, new DataflowLinkOptions() { PropagateCompletion = true }); }
public RebuildStatus Rebuild() { if (Logger.IsInfoEnabled) { Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId); } DumpProjections(); _eventUnwinder.Unwind(); _status = new RebuildStatus(); TenantContext.Enter(_config.TenantId); ConfigureProjections(); var allSlots = _projectionsBySlot.Keys.ToArray(); //initialize dispatching of the commits foreach (var bucket in _config.BucketInfo) { _consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>()); _status.AddBucket(); } //Setup the slots foreach (var slotName in allSlots) { var startCheckpoint = GetStartCheckpointForSlot(slotName); Logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint); var projectionsForThisSlot = _projectionsBySlot[slotName]; Int64 maximumDispatchedValue = projectionsForThisSlot .Select(p => _checkpointTracker.GetCheckpoint(p)) .Max(); var dispatcher = new RebuildProjectionSlotDispatcher(_logger, slotName, _config, projectionsForThisSlot, _checkpointTracker, maximumDispatchedValue); MetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch); _rebuildDispatchers.Add(dispatcher); //find right consumer var slotBucket = _config.BucketInfo.SingleOrDefault(b => b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ?? _config.BucketInfo.Single(b => b.Slots[0] == "*"); var consumerList = _consumers[slotBucket]; consumerList.Add(dispatcher); } //now start tpl and start polling in other threads. foreach (var consumer in _consumers) { var bucketInfo = String.Join(",", consumer.Key.Slots); if (consumer.Value.Count == 0) { _logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo); _status.BucketDone(bucketInfo, 0, 0, 0); continue; } DataflowBlockOptions consumerBufferOptions = new DataflowBlockOptions(); consumerBufferOptions.BoundedCapacity = _bufferSize; var _buffer = new BufferBlock <DomainEvent>(consumerBufferOptions); ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions(); executionOption.BoundedCapacity = _bufferSize; var dispatcherList = consumer.Value; _projectionInspector.ResetHandledEvents(); List <ActionBlock <DomainEvent> > consumers = new List <ActionBlock <DomainEvent> >(); foreach (var dispatcher in dispatcherList) { ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions(); consumerOptions.BoundedCapacity = _bufferSize; var actionBlock = new ActionBlock <DomainEvent>((Action <DomainEvent>)dispatcher.DispatchEvent, consumerOptions); consumers.Add(actionBlock); foreach (var projection in dispatcher.Projections) { _projectionInspector.InspectProjectionForEvents(projection.GetType()); } } var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList(); var _broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, 3000); _buffer.LinkTo(_broadcaster, new DataflowLinkOptions() { PropagateCompletion = true }); Task.Factory.StartNew(() => StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers)); } MetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit); return(_status); }