private async Task InitAsync() { _maxDispatchedCheckpoint = 0; DumpProjections(); TenantContext.Enter(_config.TenantId); await _housekeeper.InitAsync().ConfigureAwait(false); await ConfigureProjectionsAsync().ConfigureAwait(false); // cleanup await _housekeeper.RemoveAllAsync(_persistence).ConfigureAwait(false); var allSlots = _projectionsBySlot.Keys.ToArray(); var allClients = new List <ICommitPollingClient>(); //recreate all polling clients. foreach (var bucket in _config.BucketInfo) { string pollerId = "bucket: " + String.Join(",", bucket.Slots); var client = _pollingClientFactory.Create(_persistence, pollerId); allClients.Add(client); _bucketToClient.Add(bucket, client); client.Configure(GetStartGlobalCheckpoint(bucket.Slots), 4000); } _clients = allClients.ToArray(); foreach (var slotName in allSlots) { KernelMetricsHelper.CreateMeterForDispatcherCountSlot(slotName); var startCheckpoint = GetStartCheckpointForSlot(slotName); _logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint); var name = slotName; //find right consumer var slotBucket = _config.BucketInfo.SingleOrDefault(b => b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ?? _config.BucketInfo.Single(b => b.Slots[0] == "*"); var client = _bucketToClient[slotBucket]; client.AddConsumer($"SLOT: {slotName}", commit => DispatchCommitAsync(commit, name, startCheckpoint)); } Initialized = true; KernelMetricsHelper.SetProjectionEngineCurrentDispatchCount(() => _countOfConcurrentDispatchingCommit); }
/// <summary> /// Start rebuild process. /// </summary> /// <returns></returns> public async Task <RebuildStatus> RebuildAsync() { if (Logger.IsInfoEnabled) { Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId); } DumpProjections(); await _eventUnwinder.UnwindAsync().ConfigureAwait(false); _status = new RebuildStatus(); TenantContext.Enter(_config.TenantId); await ConfigureProjections().ConfigureAwait(false); var allSlots = _projectionsBySlot.Keys.ToArray(); //initialize dispatching of the commits foreach (var bucket in _config.BucketInfo) { _consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>()); _status.AddBucket(); } //Setup the slots foreach (var slotName in allSlots) { Logger.InfoFormat("Slot {0} will be rebuilded", slotName); var projectionsForThisSlot = _projectionsBySlot[slotName]; Int64 maximumDispatchedValue = projectionsForThisSlot .Select(p => _checkpointTracker.GetCheckpoint(p)) .Max(); var dispatcher = new RebuildProjectionSlotDispatcher(Logger, slotName, _config, projectionsForThisSlot, maximumDispatchedValue, _loggerThreadContextManager); KernelMetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch); _rebuildDispatchers.Add(dispatcher); //find right consumer var slotBucket = _config.BucketInfo.SingleOrDefault(b => b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ?? _config.BucketInfo.Single(b => b.Slots[0] == "*"); var consumerList = _consumers[slotBucket]; consumerList.Add(dispatcher); } //Creates TPL chain and start polling everything on the consumer foreach (var consumer in _consumers) { var bucketInfo = String.Join(",", consumer.Key.Slots); if (consumer.Value.Count == 0) { Logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo); _status.BucketDone(bucketInfo, 0, 0, 0); continue; } var consumerBufferOptions = new DataflowBlockOptions(); consumerBufferOptions.BoundedCapacity = consumer.Key.BufferSize; var _buffer = new BufferBlock <UnwindedDomainEvent>(consumerBufferOptions); ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions(); executionOption.BoundedCapacity = consumer.Key.BufferSize; var dispatcherList = consumer.Value; _projectionInspector.ResetHandledEvents(); List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> > consumers = new List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> >(); foreach (var dispatcher in dispatcherList) { ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions(); consumerOptions.BoundedCapacity = consumer.Key.BufferSize; var actionBlock = new ActionBlock <UnwindedDomainEvent>((Func <UnwindedDomainEvent, Task>)dispatcher.DispatchEventAsync, consumerOptions); HashSet <Type> eventsOfThisSlot = new HashSet <Type>(); foreach (var projection in dispatcher.Projections) { var domainEventTypesHandledByThisProjection = _projectionInspector.InspectProjectionForEvents(projection.GetType()); foreach (var type in domainEventTypesHandledByThisProjection) { eventsOfThisSlot.Add(type); } } SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> slotInfo = new SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent>( actionBlock, dispatcher.SlotName, eventsOfThisSlot); consumers.Add(slotInfo); KernelMetricsHelper.CreateMeterForRebuildDispatcherBuffer(dispatcher.SlotName, () => actionBlock.InputCount); } var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList(); var broadcaster = SlotGuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, consumer.Key.BufferSize); _buffer.LinkTo(broadcaster, new DataflowLinkOptions() { PropagateCompletion = true }); KernelMetricsHelper.CreateGaugeForRebuildFirstBuffer(bucketInfo, () => _buffer.Count); KernelMetricsHelper.CreateGaugeForRebuildBucketDBroadcasterBuffer(bucketInfo, () => broadcaster.InputCount); KernelMetricsHelper.CreateMeterForRebuildEventCompleted(bucketInfo); //fire each bucket in own thread #pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed Task.Factory.StartNew(() => StartPoll( _buffer, broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers)); //await StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers).ConfigureAwait(false); #pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed } KernelMetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit); return(_status); }