public void SetUp(IProjection[] projections, int version, Boolean setupMetrics) { Checkpoint versionInfo = _checkpoints.FindOneById("VERSION") ?? new Checkpoint("VERSION", 0, null); int currentVersion = (Int32)versionInfo.Value; Int64 projectionStartFormCheckpointValue = 0L; if (currentVersion == 0) { var eventsVersion = _checkpoints.FindOneById(EventStoreFactory.PartitionCollectionName); if (eventsVersion != null) { projectionStartFormCheckpointValue = eventsVersion.Value; } } //set all projection to active = false _checkpoints.UpdateMany( Builders <Checkpoint> .Filter.Ne(c => c.Slot, null), Builders <Checkpoint> .Update.Set(c => c.Active, false) ); foreach (var projection in projections) { Add(projection, projectionStartFormCheckpointValue); } // mark db if (version > currentVersion) { versionInfo.Value = version; _checkpoints.Save(versionInfo, versionInfo.Id); } foreach (var slot in projections.Select(p => p.Info.SlotName).Distinct()) { var slotName = slot; if (setupMetrics) { KernelMetricsHelper.SetCheckpointCountToDispatch(slot, () => GetCheckpointCount(slotName)); } _checkpointSlotTracker[slot] = 0; _slotRebuildTracker[slot] = false; } if (setupMetrics) { KernelMetricsHelper.SetCheckpointCountToDispatch("", GetCheckpointMaxCount); } }
/// <summary> /// Start rebuild process. /// </summary> /// <returns></returns> public async Task <RebuildStatus> RebuildAsync() { if (Logger.IsInfoEnabled) { Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId); } DumpProjections(); await _eventUnwinder.UnwindAsync().ConfigureAwait(false); _status = new RebuildStatus(); TenantContext.Enter(_config.TenantId); await ConfigureProjections().ConfigureAwait(false); var allSlots = _projectionsBySlot.Keys.ToArray(); //initialize dispatching of the commits foreach (var bucket in _config.BucketInfo) { _consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>()); _status.AddBucket(); } //Setup the slots foreach (var slotName in allSlots) { Logger.InfoFormat("Slot {0} will be rebuilded", slotName); var projectionsForThisSlot = _projectionsBySlot[slotName]; Int64 maximumDispatchedValue = projectionsForThisSlot .Select(p => _checkpointTracker.GetCheckpoint(p)) .Max(); var dispatcher = new RebuildProjectionSlotDispatcher(Logger, slotName, _config, projectionsForThisSlot, maximumDispatchedValue, _loggerThreadContextManager); KernelMetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch); _rebuildDispatchers.Add(dispatcher); //find right consumer var slotBucket = _config.BucketInfo.SingleOrDefault(b => b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ?? _config.BucketInfo.Single(b => b.Slots[0] == "*"); var consumerList = _consumers[slotBucket]; consumerList.Add(dispatcher); } //Creates TPL chain and start polling everything on the consumer foreach (var consumer in _consumers) { var bucketInfo = String.Join(",", consumer.Key.Slots); if (consumer.Value.Count == 0) { Logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo); _status.BucketDone(bucketInfo, 0, 0, 0); continue; } var consumerBufferOptions = new DataflowBlockOptions(); consumerBufferOptions.BoundedCapacity = consumer.Key.BufferSize; var _buffer = new BufferBlock <UnwindedDomainEvent>(consumerBufferOptions); ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions(); executionOption.BoundedCapacity = consumer.Key.BufferSize; var dispatcherList = consumer.Value; _projectionInspector.ResetHandledEvents(); List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> > consumers = new List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> >(); foreach (var dispatcher in dispatcherList) { ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions(); consumerOptions.BoundedCapacity = consumer.Key.BufferSize; var actionBlock = new ActionBlock <UnwindedDomainEvent>((Func <UnwindedDomainEvent, Task>)dispatcher.DispatchEventAsync, consumerOptions); HashSet <Type> eventsOfThisSlot = new HashSet <Type>(); foreach (var projection in dispatcher.Projections) { var domainEventTypesHandledByThisProjection = _projectionInspector.InspectProjectionForEvents(projection.GetType()); foreach (var type in domainEventTypesHandledByThisProjection) { eventsOfThisSlot.Add(type); } } SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> slotInfo = new SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent>( actionBlock, dispatcher.SlotName, eventsOfThisSlot); consumers.Add(slotInfo); KernelMetricsHelper.CreateMeterForRebuildDispatcherBuffer(dispatcher.SlotName, () => actionBlock.InputCount); } var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList(); var broadcaster = SlotGuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, consumer.Key.BufferSize); _buffer.LinkTo(broadcaster, new DataflowLinkOptions() { PropagateCompletion = true }); KernelMetricsHelper.CreateGaugeForRebuildFirstBuffer(bucketInfo, () => _buffer.Count); KernelMetricsHelper.CreateGaugeForRebuildBucketDBroadcasterBuffer(bucketInfo, () => broadcaster.InputCount); KernelMetricsHelper.CreateMeterForRebuildEventCompleted(bucketInfo); //fire each bucket in own thread #pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed Task.Factory.StartNew(() => StartPoll( _buffer, broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers)); //await StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers).ConfigureAwait(false); #pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed } KernelMetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit); return(_status); }