/// <summary>
        /// Start rebuild process.
        /// </summary>
        /// <returns></returns>
        public async Task <RebuildStatus> RebuildAsync()
        {
            if (Logger.IsInfoEnabled)
            {
                Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId);
            }
            DumpProjections();

            await _eventUnwinder.UnwindAsync().ConfigureAwait(false);

            _status = new RebuildStatus();
            TenantContext.Enter(_config.TenantId);

            await ConfigureProjections().ConfigureAwait(false);

            var allSlots = _projectionsBySlot.Keys.ToArray();

            //initialize dispatching of the commits
            foreach (var bucket in _config.BucketInfo)
            {
                _consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>());
                _status.AddBucket();
            }

            //Setup the slots
            foreach (var slotName in allSlots)
            {
                Logger.InfoFormat("Slot {0} will be rebuilded", slotName);

                var   projectionsForThisSlot = _projectionsBySlot[slotName];
                Int64 maximumDispatchedValue = projectionsForThisSlot
                                               .Select(p => _checkpointTracker.GetCheckpoint(p))
                                               .Max();
                var dispatcher = new RebuildProjectionSlotDispatcher(Logger, slotName, _config, projectionsForThisSlot, maximumDispatchedValue, _loggerThreadContextManager);
                KernelMetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch);
                _rebuildDispatchers.Add(dispatcher);

                //find right consumer
                var slotBucket = _config.BucketInfo.SingleOrDefault(b =>
                                                                    b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ??
                                 _config.BucketInfo.Single(b => b.Slots[0] == "*");
                var consumerList = _consumers[slotBucket];
                consumerList.Add(dispatcher);
            }

            //Creates TPL chain and start polling everything on the consumer
            foreach (var consumer in _consumers)
            {
                var bucketInfo = String.Join(",", consumer.Key.Slots);

                if (consumer.Value.Count == 0)
                {
                    Logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo);
                    _status.BucketDone(bucketInfo, 0, 0, 0);
                    continue;
                }
                var consumerBufferOptions = new DataflowBlockOptions();
                consumerBufferOptions.BoundedCapacity = consumer.Key.BufferSize;
                var _buffer = new BufferBlock <UnwindedDomainEvent>(consumerBufferOptions);

                ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions();
                executionOption.BoundedCapacity = consumer.Key.BufferSize;

                var dispatcherList = consumer.Value;
                _projectionInspector.ResetHandledEvents();
                List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> > consumers =
                    new List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> >();
                foreach (var dispatcher in dispatcherList)
                {
                    ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions();
                    consumerOptions.BoundedCapacity = consumer.Key.BufferSize;
                    var            actionBlock      = new ActionBlock <UnwindedDomainEvent>((Func <UnwindedDomainEvent, Task>)dispatcher.DispatchEventAsync, consumerOptions);
                    HashSet <Type> eventsOfThisSlot = new HashSet <Type>();
                    foreach (var projection in dispatcher.Projections)
                    {
                        var domainEventTypesHandledByThisProjection = _projectionInspector.InspectProjectionForEvents(projection.GetType());
                        foreach (var type in domainEventTypesHandledByThisProjection)
                        {
                            eventsOfThisSlot.Add(type);
                        }
                    }

                    SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> slotInfo =
                        new SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent>(
                            actionBlock,
                            dispatcher.SlotName,
                            eventsOfThisSlot);
                    consumers.Add(slotInfo);
                    KernelMetricsHelper.CreateMeterForRebuildDispatcherBuffer(dispatcher.SlotName, () => actionBlock.InputCount);
                }
                var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList();

                var broadcaster = SlotGuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, consumer.Key.BufferSize);
                _buffer.LinkTo(broadcaster, new DataflowLinkOptions()
                {
                    PropagateCompletion = true
                });

                KernelMetricsHelper.CreateGaugeForRebuildFirstBuffer(bucketInfo, () => _buffer.Count);
                KernelMetricsHelper.CreateGaugeForRebuildBucketDBroadcasterBuffer(bucketInfo, () => broadcaster.InputCount);

                KernelMetricsHelper.CreateMeterForRebuildEventCompleted(bucketInfo);

                //fire each bucket in own thread
#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
                Task.Factory.StartNew(() => StartPoll(
                                          _buffer,
                                          broadcaster,
                                          bucketInfo,
                                          dispatcherList,
                                          allTypeHandledStringList,
                                          consumers));

                //await StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers).ConfigureAwait(false);
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
            }

            KernelMetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
            return(_status);
        }
示例#2
0
        public RebuildStatus Rebuild()
        {
            if (Logger.IsInfoEnabled)
            {
                Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId);
            }
            DumpProjections();

            _eventUnwinder.Unwind();

            _status = new RebuildStatus();
            TenantContext.Enter(_config.TenantId);

            ConfigureProjections();

            var allSlots = _projectionsBySlot.Keys.ToArray();

            //initialize dispatching of the commits
            foreach (var bucket in _config.BucketInfo)
            {
                _consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>());
                _status.AddBucket();
            }
            //Setup the slots
            foreach (var slotName in allSlots)
            {
                var startCheckpoint = GetStartCheckpointForSlot(slotName);
                Logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint);

                var   projectionsForThisSlot = _projectionsBySlot[slotName];
                Int64 maximumDispatchedValue = projectionsForThisSlot
                                               .Select(p => _checkpointTracker.GetCheckpoint(p))
                                               .Max();
                var dispatcher = new RebuildProjectionSlotDispatcher(_logger, slotName, _config, projectionsForThisSlot, _checkpointTracker, maximumDispatchedValue);
                MetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch);
                _rebuildDispatchers.Add(dispatcher);

                //find right consumer
                var slotBucket = _config.BucketInfo.SingleOrDefault(b =>
                                                                    b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ??
                                 _config.BucketInfo.Single(b => b.Slots[0] == "*");
                var consumerList = _consumers[slotBucket];
                consumerList.Add(dispatcher);
            }

            //now start tpl and start polling in other threads.
            foreach (var consumer in _consumers)
            {
                var bucketInfo = String.Join(",", consumer.Key.Slots);

                if (consumer.Value.Count == 0)
                {
                    _logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo);
                    _status.BucketDone(bucketInfo, 0, 0, 0);
                    continue;
                }
                DataflowBlockOptions consumerBufferOptions = new DataflowBlockOptions();
                consumerBufferOptions.BoundedCapacity = _bufferSize;
                var _buffer = new BufferBlock <DomainEvent>(consumerBufferOptions);

                ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions();
                executionOption.BoundedCapacity = _bufferSize;


                var dispatcherList = consumer.Value;
                _projectionInspector.ResetHandledEvents();
                List <ActionBlock <DomainEvent> > consumers = new List <ActionBlock <DomainEvent> >();
                foreach (var dispatcher in dispatcherList)
                {
                    ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions();
                    consumerOptions.BoundedCapacity = _bufferSize;
                    var actionBlock = new ActionBlock <DomainEvent>((Action <DomainEvent>)dispatcher.DispatchEvent, consumerOptions);
                    consumers.Add(actionBlock);
                    foreach (var projection in dispatcher.Projections)
                    {
                        _projectionInspector.InspectProjectionForEvents(projection.GetType());
                    }
                }
                var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList();

                var _broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, 3000);
                _buffer.LinkTo(_broadcaster, new DataflowLinkOptions()
                {
                    PropagateCompletion = true
                });

                Task.Factory.StartNew(() => StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers));
            }

            MetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
            return(_status);
        }