Пример #1
0
        private void CreateTplChain(ref BufferBlock <AtomicDispatchChunk> buffer, ref ITargetBlock <AtomicDispatchChunk> broadcaster)
        {
            DataflowBlockOptions bufferOptions = new DataflowBlockOptions
            {
                BoundedCapacity = 4000,
            };
            var localBuffer = buffer = new BufferBlock <AtomicDispatchChunk>(bufferOptions);

            Metric.Gauge("atomic-projection-buffer", () => localBuffer.Count, Unit.Items);

            ExecutionDataflowBlockOptions enhancerExecutionOptions = new ExecutionDataflowBlockOptions
            {
                BoundedCapacity        = 4000,
                MaxDegreeOfParallelism = 1,
            };
            var enhancer = new TransformBlock <AtomicDispatchChunk, AtomicDispatchChunk>(c =>
            {
                _commitEnhancer.Enhance(c.Chunk);
                return(c);
            }, enhancerExecutionOptions);

            buffer.LinkTo(enhancer, new DataflowLinkOptions()
            {
                PropagateCompletion = true
            });
            Metric.Gauge("atomic-projection-enhancer-buffer", () => enhancer.InputCount, Unit.Items);

            ExecutionDataflowBlockOptions consumerExecutionOptions = new ExecutionDataflowBlockOptions
            {
                BoundedCapacity        = 15000,
                MaxDegreeOfParallelism = 1,
            };
            var consumers = new List <ITargetBlock <AtomicDispatchChunk> >();

            foreach (var item in _consumerBlocks)
            {
                //Ok I have a list of atomic projection, we need to create projector for every item.
                var blocks   = item.Value;
                var consumer = new ActionBlock <AtomicDispatchChunk>(InnerDispatch(item, blocks), consumerExecutionOptions);
                Metric.Gauge("atomic-projection-consumer-buffer-" + item.Key.Name, () => consumer.InputCount, Unit.Items);
                KernelMetricsHelper.CreateMeterForAtomicReadmodelDispatcherCount(item.Key.Name);
                consumers.Add(consumer);
            }
            broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, "AtomicPoller", 3000);
            enhancer.LinkTo(broadcaster, new DataflowLinkOptions()
            {
                PropagateCompletion = true
            });
        }
Пример #2
0
        private void CreateTplChain()
        {
            DataflowBlockOptions bufferOptions = new DataflowBlockOptions();

            bufferOptions.BoundedCapacity = _bufferSize;
            _buffer = new BufferBlock <IChunk>(bufferOptions);

            ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions();

            executionOption.BoundedCapacity = _bufferSize;

            _broadcaster = GuaranteedDeliveryBroadcastBlock.Create(_consumers, _id, 3000);

            _buffer.LinkTo(_broadcaster, new DataflowLinkOptions()
            {
                PropagateCompletion = true
            });
        }
Пример #3
0
        private void CreateTplChain(ref BufferBlock <AtomicDispatchChunk> buffer, ref ITargetBlock <AtomicDispatchChunk> broadcaster)
        {
            DataflowBlockOptions bufferOptions = new DataflowBlockOptions
            {
                BoundedCapacity = 4000,
            };
            var localBuffer = buffer = new BufferBlock <AtomicDispatchChunk>(bufferOptions);

            Metrics.Metric.Gauge("atomic-projection-buffer", () => localBuffer.Count, Metrics.Unit.Items);

            ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions
            {
                BoundedCapacity = 4000,
            };
            var enhancer = new TransformBlock <AtomicDispatchChunk, AtomicDispatchChunk>(c =>
            {
                _commitEnhancer.Enhance(c.Chunk); //This was already done by the poller, but we are investigating on using directly a NStore poller.
                return(c);
            }, executionOption);

            buffer.LinkTo(enhancer, new DataflowLinkOptions()
            {
                PropagateCompletion = true
            });
            Metrics.Metric.Gauge("atomic-projection-enhancer-buffer", () => (Double)enhancer.InputCount, Metrics.Unit.Items);

            var consumers = new List <ITargetBlock <AtomicDispatchChunk> >();

            foreach (var item in _consumerBlocks)
            {
                //Ok I have a list of atomic projection, we need to create projector for every item.
                var blocks   = item.Value;
                var consumer = new ActionBlock <AtomicDispatchChunk>(InnerDispatch(item, blocks), executionOption);
                Metrics.Metric.Gauge("atomic-projection-consumer-buffer-" + item.Key.Name, () => consumer.InputCount, Metrics.Unit.Items);
                consumers.Add(consumer);
            }
            broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, "AtomicPoller", 3000);
            enhancer.LinkTo(broadcaster, new DataflowLinkOptions()
            {
                PropagateCompletion = true
            });
        }
Пример #4
0
            public PollingObserveCommitsTpl(
                IPersistStreams persistStreams,
                int interval,
                string checkpointToken,
                ICommitEnhancer enhancer,
                bool boost)
            {
                _persistStreams  = persistStreams;
                _checkpointToken = checkpointToken;
                _enhancer        = enhancer;
                _interval        = interval;
                _boost           = boost;

                DataflowBlockOptions bufferOptions = new DataflowBlockOptions();

                if (_boost)
                {
                    bufferOptions.BoundedCapacity = 1000;
                }
                _buffer = new BufferBlock <ICommit>(bufferOptions);

                ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions();

                executionOption.BoundedCapacity        = 1000;
                executionOption.MaxDegreeOfParallelism = 16;
                _enhancerBlock = new TransformBlock <ICommit, ICommit>((Func <ICommit, ICommit>)Enhance, executionOption);

                _subscribers = new ConcurrentDictionary <Guid, ITargetBlock <ICommit> >();

                _broadcaster = GuaranteedDeliveryBroadcastBlock.Create(_subscribers, 1000);

                _buffer.LinkTo(_enhancerBlock, new DataflowLinkOptions()
                {
                    PropagateCompletion = true
                });
                _enhancerBlock.LinkTo(_broadcaster, new DataflowLinkOptions()
                {
                    PropagateCompletion = true
                });
            }
Пример #5
0
        public RebuildStatus Rebuild()
        {
            if (Logger.IsInfoEnabled)
            {
                Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId);
            }
            DumpProjections();

            _eventUnwinder.Unwind();

            _status = new RebuildStatus();
            TenantContext.Enter(_config.TenantId);

            ConfigureProjections();

            var allSlots = _projectionsBySlot.Keys.ToArray();

            //initialize dispatching of the commits
            foreach (var bucket in _config.BucketInfo)
            {
                _consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>());
                _status.AddBucket();
            }
            //Setup the slots
            foreach (var slotName in allSlots)
            {
                var startCheckpoint = GetStartCheckpointForSlot(slotName);
                Logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint);

                var   projectionsForThisSlot = _projectionsBySlot[slotName];
                Int64 maximumDispatchedValue = projectionsForThisSlot
                                               .Select(p => _checkpointTracker.GetCheckpoint(p))
                                               .Max();
                var dispatcher = new RebuildProjectionSlotDispatcher(_logger, slotName, _config, projectionsForThisSlot, _checkpointTracker, maximumDispatchedValue);
                MetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch);
                _rebuildDispatchers.Add(dispatcher);

                //find right consumer
                var slotBucket = _config.BucketInfo.SingleOrDefault(b =>
                                                                    b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ??
                                 _config.BucketInfo.Single(b => b.Slots[0] == "*");
                var consumerList = _consumers[slotBucket];
                consumerList.Add(dispatcher);
            }

            //now start tpl and start polling in other threads.
            foreach (var consumer in _consumers)
            {
                var bucketInfo = String.Join(",", consumer.Key.Slots);

                if (consumer.Value.Count == 0)
                {
                    _logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo);
                    _status.BucketDone(bucketInfo, 0, 0, 0);
                    continue;
                }
                DataflowBlockOptions consumerBufferOptions = new DataflowBlockOptions();
                consumerBufferOptions.BoundedCapacity = _bufferSize;
                var _buffer = new BufferBlock <DomainEvent>(consumerBufferOptions);

                ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions();
                executionOption.BoundedCapacity = _bufferSize;


                var dispatcherList = consumer.Value;
                _projectionInspector.ResetHandledEvents();
                List <ActionBlock <DomainEvent> > consumers = new List <ActionBlock <DomainEvent> >();
                foreach (var dispatcher in dispatcherList)
                {
                    ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions();
                    consumerOptions.BoundedCapacity = _bufferSize;
                    var actionBlock = new ActionBlock <DomainEvent>((Action <DomainEvent>)dispatcher.DispatchEvent, consumerOptions);
                    consumers.Add(actionBlock);
                    foreach (var projection in dispatcher.Projections)
                    {
                        _projectionInspector.InspectProjectionForEvents(projection.GetType());
                    }
                }
                var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList();

                var _broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, 3000);
                _buffer.LinkTo(_broadcaster, new DataflowLinkOptions()
                {
                    PropagateCompletion = true
                });

                Task.Factory.StartNew(() => StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers));
            }

            MetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
            return(_status);
        }