private async Task StartPoll(
BufferBlock <UnwindedDomainEvent> buffer,
ActionBlock <UnwindedDomainEvent> broadcaster,
String bucketInfo,
List <RebuildProjectionSlotDispatcher> dispatchers,
List <String> allEventTypesHandledByAllSlots,
List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> > consumers)
{
if (buffer == null)
{
throw new ArgumentNullException(nameof(buffer));
}
if (broadcaster == null)
{
throw new ArgumentNullException(nameof(broadcaster));
}
if (dispatchers == null)
{
throw new ArgumentNullException(nameof(dispatchers));
}
if (consumers == null)
{
throw new ArgumentNullException(nameof(consumers));
}
if (allEventTypesHandledByAllSlots == null)
{
throw new ArgumentNullException(nameof(allEventTypesHandledByAllSlots));
}
Stopwatch sw = Stopwatch.StartNew();
var eventCollection = _eventUnwinder.UnwindedCollection;
Int64 lastCheckpointTokenDispatched = 0;
Int32 lastSequenceDispatched = 0;
Int64 maxEventDispatched = dispatchers.Max(d => d.LastCheckpointDispatched);
Int64 totalEventToDispatch = eventCollection.Count(Builders <UnwindedDomainEvent> .Filter.Lte(e => e.CheckpointToken, maxEventDispatched));
Int64 dispatchedEvents = 0;
//this is the main cycle that continue to poll events and send to the tpl buffer
Boolean done = false;
while (!done)
{
try
{
IFindFluent <UnwindedDomainEvent, UnwindedDomainEvent> filter = eventCollection
.Find(
//Greater than last dispatched, less than maximum dispatched and one of the events that are elaborated by at least one of the projection
Builders <UnwindedDomainEvent> .Filter.And(
Builders <UnwindedDomainEvent> .Filter.Or( //this is the condition on last dispatched
Builders <UnwindedDomainEvent> .Filter.Gt(e => e.CheckpointToken, lastCheckpointTokenDispatched), //or checkpoint token is greater than last dispatched.
Builders <UnwindedDomainEvent> .Filter.And( //or is equal to the last dispatched, but the EventSequence is greater.
Builders <UnwindedDomainEvent> .Filter.Gt(e => e.EventSequence, lastSequenceDispatched),
Builders <UnwindedDomainEvent> .Filter.Eq(e => e.CheckpointToken, lastCheckpointTokenDispatched)
)
),
Builders <UnwindedDomainEvent> .Filter.Lte(e => e.CheckpointToken, maxEventDispatched) //less than or equal max event dispatched.
, Builders <UnwindedDomainEvent> .Filter.In(e => e.EventType, allEventTypesHandledByAllSlots)
)
);
Logger.InfoFormat($"polled rebuild query {filter}");
var query = filter
.Sort(Builders <UnwindedDomainEvent> .Sort.Ascending(e => e.CheckpointToken).Ascending(e => e.EventSequence));
await query.ForEachAsync(Dispatch).ConfigureAwait(false);
async Task Dispatch(UnwindedDomainEvent eventUnwinded)
{
_pollError = null;
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("TPL queued event {0}/{1} for bucket {2}", eventUnwinded.CheckpointToken, eventUnwinded.EventSequence, bucketInfo);
}
//rehydrate the event before dispatching
eventUnwinded.EnhanceEvent();
await buffer.SendAsync(eventUnwinded).ConfigureAwait(false);
dispatchedEvents++;
lastSequenceDispatched = eventUnwinded.EventSequence;
lastCheckpointTokenDispatched = eventUnwinded.CheckpointToken;
}
//Send marker unwindws domain event to signal that rebuild is finished
await buffer.SendAsync(UnwindedDomainEvent.LastEvent).ConfigureAwait(false);
done = true;
}
catch (MongoException ex)
{
_pollError = $"Unable to poll event from UnwindedCollection: {ex.Message} - Last good checkpoint dispatched { lastCheckpointTokenDispatched }";
Logger.WarnFormat(ex, "Unable to poll event from UnwindedCollection: {0} - Last good checkpoint dispatched {1}", ex.Message, lastCheckpointTokenDispatched);
}
catch (Exception ex)
{
_pollError = $"Unable to poll event from UnwindedCollection: {ex.Message} - Last good checkpoint dispatched { lastCheckpointTokenDispatched }";
Logger.FatalFormat(ex, "Unable to poll event from UnwindedCollection: {0} - Last good checkpoint dispatched {1}", ex.Message, lastCheckpointTokenDispatched);
throw;
}
}
try
{
Logger.InfoFormat("Finished loading events for bucket {0} wait for tpl to finish flush", bucketInfo);
buffer.Complete();
broadcaster.Completion.Wait(); //wait for all event to be broadcasted.
Task.WaitAll(consumers.Select(c => c.Target.Completion).ToArray()); //wait for all consumers to complete.
Thread.Sleep(1000); //wait for another secondo before finishing.
//create a list of dispatcher to wait for finish, then start waiting.
List <RebuildProjectionSlotDispatcher> dispatcherWaitingToFinish = dispatchers.ToList();
while (dispatcherWaitingToFinish.Count > 0)
{
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Waiting for {0} slot to finish events", dispatcherWaitingToFinish.Count);
}
Int32 i = dispatcherWaitingToFinish.Count - 1;
while (i >= 0)
{
var dispatcher = dispatcherWaitingToFinish[i];
if (dispatcher.Finished)
{
foreach (var projection in dispatcher.Projections)
{
try
{
await projection.StopRebuildAsync().ConfigureAwait(false);
}
catch (Exception ex)
{
Logger.ErrorFormat(ex, "Error in StopRebuild for projection {0}", projection.Info.CommonName);
throw;
}
var meter = _metrics.GetMeter(projection.Info.CommonName);
_checkpointTracker.RebuildEnded(projection, meter);
Logger.InfoFormat("Rebuild done for projection {0} @ {1}",
projection.GetType().FullName,
dispatcher.LastCheckpointDispatched
);
}
await _checkpointTracker.UpdateSlotAndSetCheckpointAsync(
dispatcher.SlotName,
dispatcher.Projections.Select(p => p.Info.CommonName),
dispatcher.LastCheckpointDispatched).ConfigureAwait(false);
dispatcherWaitingToFinish.Remove(dispatcher);
Logger.InfoFormat("Rebuild ended for slot {0}", dispatcher.SlotName);
}
i--;
}
if (dispatcherWaitingToFinish.Count > 0)
{
Thread.Sleep(2000);
}
}
sw.Stop();
Logger.InfoFormat("Bucket {0} finished dispathing all events for slots: {1}", bucketInfo, dispatchers.Select(d => d.SlotName).Aggregate((s1, s2) => s1 + ", " + s2));
_status.BucketDone(bucketInfo, dispatchedEvents, sw.ElapsedMilliseconds, totalEventToDispatch);
}
catch (Exception ex)
{
Logger.FatalFormat(ex, "Error during rebuild finish {0}", ex.Message);
throw;
}
}
private void StartPoll(
BufferBlock <DomainEvent> buffer,
ActionBlock <DomainEvent> broadcaster,
String bucketInfo,
List <RebuildProjectionSlotDispatcher> dispatchers,
List <String> allEventTypesHandledByAllSlots,
List <ActionBlock <DomainEvent> > consumers)
{
Stopwatch sw = Stopwatch.StartNew();
var eventCollection = _eventUnwinder.UnwindedCollection;
Int64 lastCheckpointTokenDispatched = 0;
Int32 lastSequenceDispatched = 0;
Int64 maxEventDispatched = dispatchers.Max(d => d.LastCheckpointDispatched);
Int64 totalEventToDispatch = eventCollection.Count(Builders <UnwindedDomainEvent> .Filter.Lte(e => e.CheckpointToken, maxEventDispatched));
Int64 dispatchedEvents = 0;
//this is the main cycle that continue to poll events and send to the tpl buffer
Boolean done = false;
while (!done)
{
try
{
var query = eventCollection
.Find(
//Greater than last dispatched, less than maximum dispatched and one of the events that are elaborated by at least one of the projection
Builders <UnwindedDomainEvent> .Filter.And(
Builders <UnwindedDomainEvent> .Filter.Or( //this is the condition on last dispatched
Builders <UnwindedDomainEvent> .Filter.Gt(e => e.CheckpointToken, lastCheckpointTokenDispatched), //or checkpoint token is greater than last dispatched.
Builders <UnwindedDomainEvent> .Filter.And( //or is equal to the last dispatched, but the EventSequence is greater.
Builders <UnwindedDomainEvent> .Filter.Gt(e => e.EventSequence, lastSequenceDispatched),
Builders <UnwindedDomainEvent> .Filter.Eq(e => e.CheckpointToken, lastCheckpointTokenDispatched)
)
),
Builders <UnwindedDomainEvent> .Filter.Lte(e => e.CheckpointToken, maxEventDispatched), //less than or equal max event dispatched.
Builders <UnwindedDomainEvent> .Filter.Or( //should be one of the handled type, or the last one.
Builders <UnwindedDomainEvent> .Filter.In(e => e.EventType, allEventTypesHandledByAllSlots),
Builders <UnwindedDomainEvent> .Filter.Eq(e => e.CheckpointToken, maxEventDispatched)
)
)
)
.Sort(Builders <UnwindedDomainEvent> .Sort.Ascending(e => e.CheckpointToken).Ascending(e => e.EventSequence));
foreach (var eventUnwinded in query.ToEnumerable())
{
var evt = eventUnwinded.GetEvent();
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("TPL queued event {0}/{1} for bucket {2}", eventUnwinded.CheckpointToken, eventUnwinded.EventSequence, bucketInfo);
}
var task = buffer.SendAsync(evt);
//wait till the commit is stored in tpl queue
task.Wait();
dispatchedEvents++;
lastSequenceDispatched = eventUnwinded.EventSequence;
lastCheckpointTokenDispatched = eventUnwinded.CheckpointToken;
}
done = true;
}
catch (MongoException ex)
{
_logger.WarnFormat(ex, "Unable to poll event from UnwindedCollection: {0} - Last good checkpoint dispatched {1}", ex.Message, lastCheckpointTokenDispatched);
}
catch (Exception ex)
{
_logger.FatalFormat(ex, "Unable to poll event from UnwindedCollection: {0} - Last good checkpoint dispatched {1}", ex.Message, lastCheckpointTokenDispatched);
}
}
try
{
_logger.InfoFormat("Finished loading events for bucket {0} wait for tpl to finish flush", bucketInfo);
buffer.Complete();
broadcaster.Completion.Wait(); //wait for all event to be broadcasted.
Task.WaitAll(consumers.Select(c => c.Completion).ToArray()); //wait for all consumers to complete.
Thread.Sleep(1000); //wait for another secondo before finishing.
//create a list of dispatcher to wait for finish
List <RebuildProjectionSlotDispatcher> dispatcherWaitingToFinish = dispatchers.ToList();
while (dispatcherWaitingToFinish.Count > 0)
{
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("Waiting for {0} slot to finish events", dispatcherWaitingToFinish.Count);
}
Int32 i = dispatcherWaitingToFinish.Count - 1;
while (i >= 0)
{
var dispatcher = dispatcherWaitingToFinish[i];
if (dispatcher.Finished)
{
foreach (var projection in dispatcher.Projections)
{
try
{
projection.StopRebuild();
}
catch (Exception ex)
{
_logger.ErrorFormat(ex, "Error in StopRebuild for projection {0}", projection.GetCommonName());
throw;
}
var meter = _metrics.GetMeter(projection.GetCommonName());
_checkpointTracker.RebuildEnded(projection, meter);
Logger.InfoFormat("Rebuild done for projection {0} @ {1}",
projection.GetType().FullName,
dispatcher.LastCheckpointDispatched
);
}
_checkpointTracker.UpdateSlotAndSetCheckpoint(
dispatcher.SlotName,
dispatcher.Projections.Select(p => p.GetCommonName()),
dispatcher.LastCheckpointDispatched);
dispatcherWaitingToFinish.Remove(dispatcher);
Logger.InfoFormat("Rebuild ended for slot {0}", dispatcher.SlotName);
}
i--;
}
Thread.Sleep(2000);
}
sw.Stop();
_logger.InfoFormat("Bucket {0} finished dispathing all events for slots: {1}", bucketInfo, dispatchers.Select(d => d.SlotName).Aggregate((s1, s2) => s1 + ", " + s2));
_status.BucketDone(bucketInfo, dispatchedEvents, sw.ElapsedMilliseconds, totalEventToDispatch);
}
catch (Exception ex)
{
_logger.FatalFormat(ex, "Error during rebuild finish {0}", ex.Message);
throw;
}
}
/// <summary>
/// Start rebuild process.
/// </summary>
/// <returns></returns>
public async Task <RebuildStatus> RebuildAsync()
{
if (Logger.IsInfoEnabled)
{
Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId);
}
DumpProjections();
await _eventUnwinder.UnwindAsync().ConfigureAwait(false);
_status = new RebuildStatus();
TenantContext.Enter(_config.TenantId);
await ConfigureProjections().ConfigureAwait(false);
var allSlots = _projectionsBySlot.Keys.ToArray();
//initialize dispatching of the commits
foreach (var bucket in _config.BucketInfo)
{
_consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>());
_status.AddBucket();
}
//Setup the slots
foreach (var slotName in allSlots)
{
Logger.InfoFormat("Slot {0} will be rebuilded", slotName);
var projectionsForThisSlot = _projectionsBySlot[slotName];
Int64 maximumDispatchedValue = projectionsForThisSlot
.Select(p => _checkpointTracker.GetCheckpoint(p))
.Max();
var dispatcher = new RebuildProjectionSlotDispatcher(Logger, slotName, _config, projectionsForThisSlot, maximumDispatchedValue, _loggerThreadContextManager);
KernelMetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch);
_rebuildDispatchers.Add(dispatcher);
//find right consumer
var slotBucket = _config.BucketInfo.SingleOrDefault(b =>
b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ??
_config.BucketInfo.Single(b => b.Slots[0] == "*");
var consumerList = _consumers[slotBucket];
consumerList.Add(dispatcher);
}
//Creates TPL chain and start polling everything on the consumer
foreach (var consumer in _consumers)
{
var bucketInfo = String.Join(",", consumer.Key.Slots);
if (consumer.Value.Count == 0)
{
Logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo);
_status.BucketDone(bucketInfo, 0, 0, 0);
continue;
}
var consumerBufferOptions = new DataflowBlockOptions();
consumerBufferOptions.BoundedCapacity = consumer.Key.BufferSize;
var _buffer = new BufferBlock <UnwindedDomainEvent>(consumerBufferOptions);
ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions();
executionOption.BoundedCapacity = consumer.Key.BufferSize;
var dispatcherList = consumer.Value;
_projectionInspector.ResetHandledEvents();
List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> > consumers =
new List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> >();
foreach (var dispatcher in dispatcherList)
{
ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions();
consumerOptions.BoundedCapacity = consumer.Key.BufferSize;
var actionBlock = new ActionBlock <UnwindedDomainEvent>((Func <UnwindedDomainEvent, Task>)dispatcher.DispatchEventAsync, consumerOptions);
HashSet <Type> eventsOfThisSlot = new HashSet <Type>();
foreach (var projection in dispatcher.Projections)
{
var domainEventTypesHandledByThisProjection = _projectionInspector.InspectProjectionForEvents(projection.GetType());
foreach (var type in domainEventTypesHandledByThisProjection)
{
eventsOfThisSlot.Add(type);
}
}
SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> slotInfo =
new SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent>(
actionBlock,
dispatcher.SlotName,
eventsOfThisSlot);
consumers.Add(slotInfo);
KernelMetricsHelper.CreateMeterForRebuildDispatcherBuffer(dispatcher.SlotName, () => actionBlock.InputCount);
}
var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList();
var broadcaster = SlotGuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, consumer.Key.BufferSize);
_buffer.LinkTo(broadcaster, new DataflowLinkOptions()
{
PropagateCompletion = true
});
KernelMetricsHelper.CreateGaugeForRebuildFirstBuffer(bucketInfo, () => _buffer.Count);
KernelMetricsHelper.CreateGaugeForRebuildBucketDBroadcasterBuffer(bucketInfo, () => broadcaster.InputCount);
KernelMetricsHelper.CreateMeterForRebuildEventCompleted(bucketInfo);
//fire each bucket in own thread
#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
Task.Factory.StartNew(() => StartPoll(
_buffer,
broadcaster,
bucketInfo,
dispatcherList,
allTypeHandledStringList,
consumers));
//await StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers).ConfigureAwait(false);
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
}
KernelMetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
return(_status);
}
public RebuildStatus Rebuild()
{
if (Logger.IsInfoEnabled)
{
Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId);
}
DumpProjections();
_eventUnwinder.Unwind();
_status = new RebuildStatus();
TenantContext.Enter(_config.TenantId);
ConfigureProjections();
var allSlots = _projectionsBySlot.Keys.ToArray();
//initialize dispatching of the commits
foreach (var bucket in _config.BucketInfo)
{
_consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>());
_status.AddBucket();
}
//Setup the slots
foreach (var slotName in allSlots)
{
var startCheckpoint = GetStartCheckpointForSlot(slotName);
Logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint);
var projectionsForThisSlot = _projectionsBySlot[slotName];
Int64 maximumDispatchedValue = projectionsForThisSlot
.Select(p => _checkpointTracker.GetCheckpoint(p))
.Max();
var dispatcher = new RebuildProjectionSlotDispatcher(_logger, slotName, _config, projectionsForThisSlot, _checkpointTracker, maximumDispatchedValue);
MetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch);
_rebuildDispatchers.Add(dispatcher);
//find right consumer
var slotBucket = _config.BucketInfo.SingleOrDefault(b =>
b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ??
_config.BucketInfo.Single(b => b.Slots[0] == "*");
var consumerList = _consumers[slotBucket];
consumerList.Add(dispatcher);
}
//now start tpl and start polling in other threads.
foreach (var consumer in _consumers)
{
var bucketInfo = String.Join(",", consumer.Key.Slots);
if (consumer.Value.Count == 0)
{
_logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo);
_status.BucketDone(bucketInfo, 0, 0, 0);
continue;
}
DataflowBlockOptions consumerBufferOptions = new DataflowBlockOptions();
consumerBufferOptions.BoundedCapacity = _bufferSize;
var _buffer = new BufferBlock <DomainEvent>(consumerBufferOptions);
ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions();
executionOption.BoundedCapacity = _bufferSize;
var dispatcherList = consumer.Value;
_projectionInspector.ResetHandledEvents();
List <ActionBlock <DomainEvent> > consumers = new List <ActionBlock <DomainEvent> >();
foreach (var dispatcher in dispatcherList)
{
ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions();
consumerOptions.BoundedCapacity = _bufferSize;
var actionBlock = new ActionBlock <DomainEvent>((Action <DomainEvent>)dispatcher.DispatchEvent, consumerOptions);
consumers.Add(actionBlock);
foreach (var projection in dispatcher.Projections)
{
_projectionInspector.InspectProjectionForEvents(projection.GetType());
}
}
var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList();
var _broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, 3000);
_buffer.LinkTo(_broadcaster, new DataflowLinkOptions()
{
PropagateCompletion = true
});
Task.Factory.StartNew(() => StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers));
}
MetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
return(_status);
}