public void SetUp(IProjection[] projections, int version, Boolean setupMetrics)
{
Checkpoint versionInfo = _checkpoints.FindOneById("VERSION")
?? new Checkpoint("VERSION", 0, null);
int currentVersion = (Int32)versionInfo.Value;
Int64 projectionStartFormCheckpointValue = 0L;
if (currentVersion == 0)
{
var eventsVersion = _checkpoints.FindOneById(EventStoreFactory.PartitionCollectionName);
if (eventsVersion != null)
{
projectionStartFormCheckpointValue = eventsVersion.Value;
}
}
//set all projection to active = false
_checkpoints.UpdateMany(
Builders <Checkpoint> .Filter.Ne(c => c.Slot, null),
Builders <Checkpoint> .Update.Set(c => c.Active, false)
);
foreach (var projection in projections)
{
Add(projection, projectionStartFormCheckpointValue);
}
// mark db
if (version > currentVersion)
{
versionInfo.Value = version;
_checkpoints.Save(versionInfo, versionInfo.Id);
}
foreach (var slot in projections.Select(p => p.Info.SlotName).Distinct())
{
var slotName = slot;
if (setupMetrics)
{
KernelMetricsHelper.SetCheckpointCountToDispatch(slot, () => GetCheckpointCount(slotName));
}
_checkpointSlotTracker[slot] = 0;
_slotRebuildTracker[slot] = false;
}
if (setupMetrics)
{
KernelMetricsHelper.SetCheckpointCountToDispatch("", GetCheckpointMaxCount);
}
}
private void CreateTplChain(ref BufferBlock <AtomicDispatchChunk> buffer, ref ITargetBlock <AtomicDispatchChunk> broadcaster)
{
DataflowBlockOptions bufferOptions = new DataflowBlockOptions
{
BoundedCapacity = 4000,
};
var localBuffer = buffer = new BufferBlock <AtomicDispatchChunk>(bufferOptions);
Metric.Gauge("atomic-projection-buffer", () => localBuffer.Count, Unit.Items);
ExecutionDataflowBlockOptions enhancerExecutionOptions = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 4000,
MaxDegreeOfParallelism = 1,
};
var enhancer = new TransformBlock <AtomicDispatchChunk, AtomicDispatchChunk>(c =>
{
_commitEnhancer.Enhance(c.Chunk);
return(c);
}, enhancerExecutionOptions);
buffer.LinkTo(enhancer, new DataflowLinkOptions()
{
PropagateCompletion = true
});
Metric.Gauge("atomic-projection-enhancer-buffer", () => enhancer.InputCount, Unit.Items);
ExecutionDataflowBlockOptions consumerExecutionOptions = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 15000,
MaxDegreeOfParallelism = 1,
};
var consumers = new List <ITargetBlock <AtomicDispatchChunk> >();
foreach (var item in _consumerBlocks)
{
//Ok I have a list of atomic projection, we need to create projector for every item.
var blocks = item.Value;
var consumer = new ActionBlock <AtomicDispatchChunk>(InnerDispatch(item, blocks), consumerExecutionOptions);
Metric.Gauge("atomic-projection-consumer-buffer-" + item.Key.Name, () => consumer.InputCount, Unit.Items);
KernelMetricsHelper.CreateMeterForAtomicReadmodelDispatcherCount(item.Key.Name);
consumers.Add(consumer);
}
broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, "AtomicPoller", 3000);
enhancer.LinkTo(broadcaster, new DataflowLinkOptions()
{
PropagateCompletion = true
});
}
private async Task InitAsync()
{
_maxDispatchedCheckpoint = 0;
DumpProjections();
TenantContext.Enter(_config.TenantId);
await _housekeeper.InitAsync().ConfigureAwait(false);
await ConfigureProjectionsAsync().ConfigureAwait(false);
// cleanup
await _housekeeper.RemoveAllAsync(_persistence).ConfigureAwait(false);
var allSlots = _projectionsBySlot.Keys.ToArray();
var allClients = new List <ICommitPollingClient>();
//recreate all polling clients.
foreach (var bucket in _config.BucketInfo)
{
string pollerId = "bucket: " + String.Join(",", bucket.Slots);
var client = _pollingClientFactory.Create(_persistence, pollerId);
allClients.Add(client);
_bucketToClient.Add(bucket, client);
client.Configure(GetStartGlobalCheckpoint(bucket.Slots), 4000);
}
_clients = allClients.ToArray();
foreach (var slotName in allSlots)
{
KernelMetricsHelper.CreateMeterForDispatcherCountSlot(slotName);
var startCheckpoint = GetStartCheckpointForSlot(slotName);
_logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint);
var name = slotName;
//find right consumer
var slotBucket = _config.BucketInfo.SingleOrDefault(b =>
b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ??
_config.BucketInfo.Single(b => b.Slots[0] == "*");
var client = _bucketToClient[slotBucket];
client.AddConsumer($"SLOT: {slotName}", commit => DispatchCommitAsync(commit, name, startCheckpoint));
}
Initialized = true;
KernelMetricsHelper.SetProjectionEngineCurrentDispatchCount(() => _countOfConcurrentDispatchingCommit);
}
public void InitSut()
{
var config = new ProjectionEngineConfig();
projections = new [] { new Projection(Substitute.For <ICollectionWrapper <SampleReadModel, String> >()) };
sut = new RebuildProjectionSlotDispatcher(
NullLogger.Instance,
slotName,
config,
projections,
4,
NullLoggerThreadContextManager.Instance);
//Needed to avoid crash on wrong metrics dispatch.
KernelMetricsHelper.CreateMeterForRebuildDispatcherBuffer(slotName, () => 0);
}
private void RegisterHealthChecks(string pollerName)
{
KernelMetricsHelper.SetCommitPollingClientBufferSize(pollerName, () => GetClientBufferSize());
//Set health check for polling
HealthChecks.RegisterHealthCheck("Polling-" + pollerName, () =>
{
if (Status == CommitPollingClientStatus.Stopped)
{
//poller is stopped, system healty
return(HealthCheckResult.Healthy("Automatic polling stopped"));
}
else if (Status == CommitPollingClientStatus.Faulted || LastException != null)
{
//poller is stopped, system is not healty anymore.
var exceptionText = (LastException != null ? LastException.ToString() : "");
exceptionText = exceptionText.Replace("{", "{{").Replace("}", "}}");
return(HealthCheckResult.Unhealthy(
"[LastDispatchedPosition: {0}] - Faulted (exception in consumer): {1}",
_lastDispatchedPosition,
exceptionText));
}
return(HealthCheckResult.Healthy("Poller alive"));
});
//Now register health check for the internal NES poller, to diagnose errors in polling.
HealthChecks.RegisterHealthCheck("Polling internal errors: ", () =>
{
if (_innerSubscription?.LastException != null)
{
return(HealthCheckResult.Unhealthy("[LastDispatchedPosition: {0}] - Inner NStore poller has error: {1}",
_lastDispatchedPosition,
_innerSubscription?.LastException));
}
return(HealthCheckResult.Healthy("Inner NES Poller Ok"));
});
}
private Func <AtomicDispatchChunk, Task> InnerDispatch(
KeyValuePair <Type, List <AtomicDispatchChunkConsumer> > item,
List <AtomicDispatchChunkConsumer> blocks)
{
return(async dispatchObj =>
{
try
{
if (dispatchObj.Chunk.Payload is Changeset changeset)
{
var aggregateId = changeset.GetIdentity();
if (aggregateId?.GetType() == item.Key)
{
//Remember to dispatch this only to objects that are associated to this poller. Remember that we build a single
//TPL chain, so both the poller (default and catchup) will push data inside the very same tpl pipeline, but inside
//this dispatcher chain, we simply dispatch only to the blocks that have correct poller id.
QueryPerformanceCounter(out long ticks1);
foreach (var atomicDispatchChunkConsumer in blocks.Where(_ => _.PollerId == dispatchObj.PollerId))
{
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Dispatched chunk {0} with poller {1} for readmodel {2}", dispatchObj.Chunk.Position, dispatchObj.PollerId, atomicDispatchChunkConsumer.Consumer.AtomicReadmodelInfoAttribute.Name);
}
//let the abstract readmodel handle everyting, then finally dispatch notification.Handle method internally
//catches exception and make atomic readmodel faulty.
var handleReturnValue = await atomicDispatchChunkConsumer.Consumer.Handle(
dispatchObj.Chunk.Position,
changeset,
aggregateId).ConfigureAwait(false);
if (handleReturnValue != null)
{
#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
//I do not care if the notification works or not, I simply call it and forget.
if (handleReturnValue.CreatedForFirstTime)
{
AtomicReadmodelNotifier.ReadmodelCreatedAsync(handleReturnValue.Readmodel, changeset);
}
else
{
AtomicReadmodelNotifier.ReadmodelUpdatedAsync(handleReturnValue.Readmodel, changeset);
}
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
}
}
QueryPerformanceCounter(out long ticks2);
var elapsedTicks = ticks2 - ticks1;
KernelMetricsHelper.IncrementProjectionCounterAtomicProjection(item.Key.Name, elapsedTicks);
KernelMetricsHelper.MarkCommitDispatchedAtomicReadmodelCount(item.Key.Name, 1);
}
}
foreach (var rm in blocks.Where(_ => _.PollerId == dispatchObj.PollerId))
{
_atomicProjectionCheckpointManager.MarkPosition(rm.Consumer.AtomicReadmodelInfoAttribute.Name, dispatchObj.Chunk.Position);
}
}
catch (Exception ex)
{
//TODO: Implement health check with failed
_engineExceptions[item.Key.Name] = ex;
Logger.ErrorFormat(ex, "Generic error in TPL dispatching {0} with poller {1} for id type {2}", dispatchObj.Chunk.Position, dispatchObj.PollerId, item.Key);
throw;
}
});
}
internal async Task DispatchEventAsync(UnwindedDomainEvent unwindedEvent)
{
if (unwindedEvent == UnwindedDomainEvent.LastEvent)
{
Finished = true;
_lastCheckpointRebuilded = LastCheckpointDispatched; //Set to zero metrics, we dispatched everything.
return;
}
var chkpoint = unwindedEvent.CheckpointToken;
if (chkpoint > LastCheckpointDispatched)
{
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("Discharded event {0} commit {1} because last checkpoint dispatched for slot {2} is {3}.", unwindedEvent.CommitId, unwindedEvent.CheckpointToken, SlotName, _maxCheckpointDispatched);
}
return;
}
Interlocked.Increment(ref RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
TenantContext.Enter(_config.TenantId);
try
{
string eventName = unwindedEvent.EventType;
foreach (var projection in _projections)
{
var cname = projection.Info.CommonName;
long elapsedticks;
try
{
QueryPerformanceCounter(out long ticks1);
await projection.HandleAsync(unwindedEvent.GetEvent(), true).ConfigureAwait(false);
QueryPerformanceCounter(out long ticks2);
elapsedticks = ticks2 - ticks1;
KernelMetricsHelper.IncrementProjectionCounterRebuild(cname, SlotName, eventName, elapsedticks);
}
catch (Exception ex)
{
_logger.FatalFormat(ex, "[Slot: {3} Projection: {4}] Failed checkpoint: {0} StreamId: {1} Event Name: {2}",
unwindedEvent.CheckpointToken,
unwindedEvent.PartitionId,
eventName,
SlotName,
cname
);
HealthChecks.RegisterHealthCheck($"RebuildDispatcher, slot {SlotName} - FailedCheckpoint {unwindedEvent.CheckpointToken}", () =>
HealthCheckResult.Unhealthy(ex)
);
throw;
}
_metrics.Inc(cname, eventName, elapsedticks);
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("[{3}] [{4}] Handled checkpoint {0}: {1} > {2}",
unwindedEvent.CheckpointToken,
unwindedEvent.PartitionId,
eventName,
SlotName,
cname
);
}
}
}
catch (Exception ex)
{
_logger.ErrorFormat(ex, "Error dispathing commit id: {0}\nMessage: {1}\nError: {2}",
unwindedEvent.CheckpointToken, unwindedEvent.Event, ex.Message);
HealthChecks.RegisterHealthCheck($"RebuildDispatcher, slot {SlotName} - GeneralError", () =>
HealthCheckResult.Unhealthy(ex)
);
throw;
}
_lastCheckpointRebuilded = chkpoint;
KernelMetricsHelper.MarkEventInRebuildDispatchedCount(SlotName, 1);
Interlocked.Decrement(ref RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
}
/// <summary>
/// Start rebuild process.
/// </summary>
/// <returns></returns>
public async Task <RebuildStatus> RebuildAsync()
{
if (Logger.IsInfoEnabled)
{
Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId);
}
DumpProjections();
await _eventUnwinder.UnwindAsync().ConfigureAwait(false);
_status = new RebuildStatus();
TenantContext.Enter(_config.TenantId);
await ConfigureProjections().ConfigureAwait(false);
var allSlots = _projectionsBySlot.Keys.ToArray();
//initialize dispatching of the commits
foreach (var bucket in _config.BucketInfo)
{
_consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>());
_status.AddBucket();
}
//Setup the slots
foreach (var slotName in allSlots)
{
Logger.InfoFormat("Slot {0} will be rebuilded", slotName);
var projectionsForThisSlot = _projectionsBySlot[slotName];
Int64 maximumDispatchedValue = projectionsForThisSlot
.Select(p => _checkpointTracker.GetCheckpoint(p))
.Max();
var dispatcher = new RebuildProjectionSlotDispatcher(Logger, slotName, _config, projectionsForThisSlot, maximumDispatchedValue, _loggerThreadContextManager);
KernelMetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch);
_rebuildDispatchers.Add(dispatcher);
//find right consumer
var slotBucket = _config.BucketInfo.SingleOrDefault(b =>
b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ??
_config.BucketInfo.Single(b => b.Slots[0] == "*");
var consumerList = _consumers[slotBucket];
consumerList.Add(dispatcher);
}
//Creates TPL chain and start polling everything on the consumer
foreach (var consumer in _consumers)
{
var bucketInfo = String.Join(",", consumer.Key.Slots);
if (consumer.Value.Count == 0)
{
Logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo);
_status.BucketDone(bucketInfo, 0, 0, 0);
continue;
}
var consumerBufferOptions = new DataflowBlockOptions();
consumerBufferOptions.BoundedCapacity = consumer.Key.BufferSize;
var _buffer = new BufferBlock <UnwindedDomainEvent>(consumerBufferOptions);
ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions();
executionOption.BoundedCapacity = consumer.Key.BufferSize;
var dispatcherList = consumer.Value;
_projectionInspector.ResetHandledEvents();
List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> > consumers =
new List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> >();
foreach (var dispatcher in dispatcherList)
{
ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions();
consumerOptions.BoundedCapacity = consumer.Key.BufferSize;
var actionBlock = new ActionBlock <UnwindedDomainEvent>((Func <UnwindedDomainEvent, Task>)dispatcher.DispatchEventAsync, consumerOptions);
HashSet <Type> eventsOfThisSlot = new HashSet <Type>();
foreach (var projection in dispatcher.Projections)
{
var domainEventTypesHandledByThisProjection = _projectionInspector.InspectProjectionForEvents(projection.GetType());
foreach (var type in domainEventTypesHandledByThisProjection)
{
eventsOfThisSlot.Add(type);
}
}
SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> slotInfo =
new SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent>(
actionBlock,
dispatcher.SlotName,
eventsOfThisSlot);
consumers.Add(slotInfo);
KernelMetricsHelper.CreateMeterForRebuildDispatcherBuffer(dispatcher.SlotName, () => actionBlock.InputCount);
}
var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList();
var broadcaster = SlotGuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, consumer.Key.BufferSize);
_buffer.LinkTo(broadcaster, new DataflowLinkOptions()
{
PropagateCompletion = true
});
KernelMetricsHelper.CreateGaugeForRebuildFirstBuffer(bucketInfo, () => _buffer.Count);
KernelMetricsHelper.CreateGaugeForRebuildBucketDBroadcasterBuffer(bucketInfo, () => broadcaster.InputCount);
KernelMetricsHelper.CreateMeterForRebuildEventCompleted(bucketInfo);
//fire each bucket in own thread
#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
Task.Factory.StartNew(() => StartPoll(
_buffer,
broadcaster,
bucketInfo,
dispatcherList,
allTypeHandledStringList,
consumers));
//await StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers).ConfigureAwait(false);
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
}
KernelMetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
return(_status);
}
private async Task DispatchCommitAsync(IChunk chunk, string slotName, Int64 startCheckpoint)
{
Interlocked.Increment(ref _countOfConcurrentDispatchingCommit);
_loggerThreadContextManager.SetContextProperty("commit", $"{chunk.OperationId}/{chunk.Position}");
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("Dispatching checkpoit {0} on tenant {1}", chunk.Position, _config.TenantId);
}
TenantContext.Enter(_config.TenantId);
var chkpoint = chunk.Position;
if (!lastCheckpointDispatched.ContainsKey(slotName))
{
lastCheckpointDispatched[slotName] = 0;
}
if (lastCheckpointDispatched[slotName] >= chkpoint)
{
var error = String.Format("Sequence broken, last checkpoint for slot {0} was {1} and now we dispatched {2}",
slotName, lastCheckpointDispatched[slotName], chkpoint);
_logger.Error(error);
throw new JarvisFrameworkEngineException(error);
}
if (lastCheckpointDispatched[slotName] + 1 != chkpoint && lastCheckpointDispatched[slotName] > 0)
{
_logger.DebugFormat("Sequence of commit not consecutive (probably holes), last dispatched {0} receiving {1}",
lastCheckpointDispatched[slotName], chkpoint);
}
lastCheckpointDispatched[slotName] = chkpoint;
if (chkpoint <= startCheckpoint)
{
//Already dispatched, skip it.
Interlocked.Decrement(ref _countOfConcurrentDispatchingCommit);
return;
}
var projections = _projectionsBySlot[slotName];
object[] events = GetArrayOfObjectFromChunk(chunk);
var eventCount = events.Length;
//now it is time to dispatch the commit, we will dispatch each projection
//and for each projection we dispatch all the events present in changeset
bool someProjectionProcessedTheEvent = false;
foreach (var projection in projections)
{
var cname = projection.Info.CommonName;
Object eventMessage = null;
try
{
//Foreach projection we need to dispach every event
for (int index = 0; index < eventCount; index++)
{
eventMessage = events[index];
var message = eventMessage as IMessage;
string eventName = eventMessage.GetType().Name;
_loggerThreadContextManager.SetContextProperty("evType", eventName);
_loggerThreadContextManager.SetContextProperty("evMsId", message?.MessageId);
_loggerThreadContextManager.SetContextProperty("evCheckpointToken", chunk.Position);
_loggerThreadContextManager.SetContextProperty("prj", cname);
var checkpointStatus = _checkpointTracker.GetCheckpointStatus(cname, chunk.Position);
long ticks = 0;
try
{
//pay attention, stopwatch consumes time.
var sw = new Stopwatch();
sw.Start();
var eventProcessed = await projection
.HandleAsync(eventMessage, checkpointStatus.IsRebuilding)
.ConfigureAwait(false);
someProjectionProcessedTheEvent |= eventProcessed;
sw.Stop();
ticks = sw.ElapsedTicks;
KernelMetricsHelper.IncrementProjectionCounter(cname, slotName, eventName, ticks, sw.ElapsedMilliseconds);
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("[Slot:{3}] [Projection {4}] Handled checkpoint {0}: {1} > {2}",
chunk.Position,
chunk.PartitionId,
$"eventName: {eventName} [event N°{index}]",
slotName,
cname
);
}
}
catch (Exception ex)
{
var error = String.Format("[Slot: {3} Projection: {4}] Failed checkpoint: {0} StreamId: {1} Event Name: {2}",
chunk.Position,
chunk.PartitionId,
eventName,
slotName,
cname);
_logger.Fatal(error, ex);
_engineFatalErrors.Add(String.Format("{0}\n{1}", error, ex));
throw;
}
} //End of event cycle
}
catch (Exception ex)
{
_loggerThreadContextManager.ClearContextProperty("commit");
_logger.ErrorFormat(ex, "Error dispathing Chunk [{0}]\n Message: {1}\n Error: {2}\n",
chunk.Position, eventMessage?.GetType()?.Name, ex.Message);
throw;
}
finally
{
_loggerThreadContextManager.ClearContextProperty("evType");
_loggerThreadContextManager.ClearContextProperty("evMsId");
_loggerThreadContextManager.ClearContextProperty("evCheckpointToken");
_loggerThreadContextManager.ClearContextProperty("prj");
}
projection.CheckpointProjected(chunk.Position);
//TODO: Evaluate if it is needed to update single projection checkpoint
//Update this projection, set all events of this checkpoint as dispatched.
//await _checkpointTracker.UpdateProjectionCheckpointAsync(cname, chunk.Position).ConfigureAwait(false);
} //End of projection cycle.
await _checkpointTracker.UpdateSlotAndSetCheckpointAsync(
slotName,
projections.Select(_ => _.Info.CommonName),
chunk.Position,
someEventDispatched : someProjectionProcessedTheEvent).ConfigureAwait(false);
KernelMetricsHelper.MarkCommitDispatchedCount(slotName, 1);
await _notifyCommitHandled.SetDispatched(slotName, chunk).ConfigureAwait(false);
// ok in multithread wihout locks!
if (_maxDispatchedCheckpoint < chkpoint)
{
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("Updating last dispatched checkpoint from {0} to {1}",
_maxDispatchedCheckpoint,
chkpoint
);
}
_maxDispatchedCheckpoint = chkpoint;
}
_loggerThreadContextManager.ClearContextProperty("commit");
Interlocked.Decrement(ref _countOfConcurrentDispatchingCommit);
}
