private void CreateTplChain(ref BufferBlock <AtomicDispatchChunk> buffer, ref ITargetBlock <AtomicDispatchChunk> broadcaster)
{
DataflowBlockOptions bufferOptions = new DataflowBlockOptions
{
BoundedCapacity = 4000,
};
var localBuffer = buffer = new BufferBlock <AtomicDispatchChunk>(bufferOptions);
Metric.Gauge("atomic-projection-buffer", () => localBuffer.Count, Unit.Items);
ExecutionDataflowBlockOptions enhancerExecutionOptions = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 4000,
MaxDegreeOfParallelism = 1,
};
var enhancer = new TransformBlock <AtomicDispatchChunk, AtomicDispatchChunk>(c =>
{
_commitEnhancer.Enhance(c.Chunk);
return(c);
}, enhancerExecutionOptions);
buffer.LinkTo(enhancer, new DataflowLinkOptions()
{
PropagateCompletion = true
});
Metric.Gauge("atomic-projection-enhancer-buffer", () => enhancer.InputCount, Unit.Items);
ExecutionDataflowBlockOptions consumerExecutionOptions = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 15000,
MaxDegreeOfParallelism = 1,
};
var consumers = new List <ITargetBlock <AtomicDispatchChunk> >();
foreach (var item in _consumerBlocks)
{
//Ok I have a list of atomic projection, we need to create projector for every item.
var blocks = item.Value;
var consumer = new ActionBlock <AtomicDispatchChunk>(InnerDispatch(item, blocks), consumerExecutionOptions);
Metric.Gauge("atomic-projection-consumer-buffer-" + item.Key.Name, () => consumer.InputCount, Unit.Items);
KernelMetricsHelper.CreateMeterForAtomicReadmodelDispatcherCount(item.Key.Name);
consumers.Add(consumer);
}
broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, "AtomicPoller", 3000);
enhancer.LinkTo(broadcaster, new DataflowLinkOptions()
{
PropagateCompletion = true
});
}
private void CreateTplChain()
{
DataflowBlockOptions bufferOptions = new DataflowBlockOptions();
bufferOptions.BoundedCapacity = _bufferSize;
_buffer = new BufferBlock <IChunk>(bufferOptions);
ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions();
executionOption.BoundedCapacity = _bufferSize;
_broadcaster = GuaranteedDeliveryBroadcastBlock.Create(_consumers, _id, 3000);
_buffer.LinkTo(_broadcaster, new DataflowLinkOptions()
{
PropagateCompletion = true
});
}
private void CreateTplChain(ref BufferBlock <AtomicDispatchChunk> buffer, ref ITargetBlock <AtomicDispatchChunk> broadcaster)
{
DataflowBlockOptions bufferOptions = new DataflowBlockOptions
{
BoundedCapacity = 4000,
};
var localBuffer = buffer = new BufferBlock <AtomicDispatchChunk>(bufferOptions);
Metrics.Metric.Gauge("atomic-projection-buffer", () => localBuffer.Count, Metrics.Unit.Items);
ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 4000,
};
var enhancer = new TransformBlock <AtomicDispatchChunk, AtomicDispatchChunk>(c =>
{
_commitEnhancer.Enhance(c.Chunk); //This was already done by the poller, but we are investigating on using directly a NStore poller.
return(c);
}, executionOption);
buffer.LinkTo(enhancer, new DataflowLinkOptions()
{
PropagateCompletion = true
});
Metrics.Metric.Gauge("atomic-projection-enhancer-buffer", () => (Double)enhancer.InputCount, Metrics.Unit.Items);
var consumers = new List <ITargetBlock <AtomicDispatchChunk> >();
foreach (var item in _consumerBlocks)
{
//Ok I have a list of atomic projection, we need to create projector for every item.
var blocks = item.Value;
var consumer = new ActionBlock <AtomicDispatchChunk>(InnerDispatch(item, blocks), executionOption);
Metrics.Metric.Gauge("atomic-projection-consumer-buffer-" + item.Key.Name, () => consumer.InputCount, Metrics.Unit.Items);
consumers.Add(consumer);
}
broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, "AtomicPoller", 3000);
enhancer.LinkTo(broadcaster, new DataflowLinkOptions()
{
PropagateCompletion = true
});
}
public PollingObserveCommitsTpl(
IPersistStreams persistStreams,
int interval,
string checkpointToken,
ICommitEnhancer enhancer,
bool boost)
{
_persistStreams = persistStreams;
_checkpointToken = checkpointToken;
_enhancer = enhancer;
_interval = interval;
_boost = boost;
DataflowBlockOptions bufferOptions = new DataflowBlockOptions();
if (_boost)
{
bufferOptions.BoundedCapacity = 1000;
}
_buffer = new BufferBlock <ICommit>(bufferOptions);
ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions();
executionOption.BoundedCapacity = 1000;
executionOption.MaxDegreeOfParallelism = 16;
_enhancerBlock = new TransformBlock <ICommit, ICommit>((Func <ICommit, ICommit>)Enhance, executionOption);
_subscribers = new ConcurrentDictionary <Guid, ITargetBlock <ICommit> >();
_broadcaster = GuaranteedDeliveryBroadcastBlock.Create(_subscribers, 1000);
_buffer.LinkTo(_enhancerBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
_enhancerBlock.LinkTo(_broadcaster, new DataflowLinkOptions()
{
PropagateCompletion = true
});
}
public RebuildStatus Rebuild()
{
if (Logger.IsInfoEnabled)
{
Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId);
}
DumpProjections();
_eventUnwinder.Unwind();
_status = new RebuildStatus();
TenantContext.Enter(_config.TenantId);
ConfigureProjections();
var allSlots = _projectionsBySlot.Keys.ToArray();
//initialize dispatching of the commits
foreach (var bucket in _config.BucketInfo)
{
_consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>());
_status.AddBucket();
}
//Setup the slots
foreach (var slotName in allSlots)
{
var startCheckpoint = GetStartCheckpointForSlot(slotName);
Logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint);
var projectionsForThisSlot = _projectionsBySlot[slotName];
Int64 maximumDispatchedValue = projectionsForThisSlot
.Select(p => _checkpointTracker.GetCheckpoint(p))
.Max();
var dispatcher = new RebuildProjectionSlotDispatcher(_logger, slotName, _config, projectionsForThisSlot, _checkpointTracker, maximumDispatchedValue);
MetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch);
_rebuildDispatchers.Add(dispatcher);
//find right consumer
var slotBucket = _config.BucketInfo.SingleOrDefault(b =>
b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ??
_config.BucketInfo.Single(b => b.Slots[0] == "*");
var consumerList = _consumers[slotBucket];
consumerList.Add(dispatcher);
}
//now start tpl and start polling in other threads.
foreach (var consumer in _consumers)
{
var bucketInfo = String.Join(",", consumer.Key.Slots);
if (consumer.Value.Count == 0)
{
_logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo);
_status.BucketDone(bucketInfo, 0, 0, 0);
continue;
}
DataflowBlockOptions consumerBufferOptions = new DataflowBlockOptions();
consumerBufferOptions.BoundedCapacity = _bufferSize;
var _buffer = new BufferBlock <DomainEvent>(consumerBufferOptions);
ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions();
executionOption.BoundedCapacity = _bufferSize;
var dispatcherList = consumer.Value;
_projectionInspector.ResetHandledEvents();
List <ActionBlock <DomainEvent> > consumers = new List <ActionBlock <DomainEvent> >();
foreach (var dispatcher in dispatcherList)
{
ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions();
consumerOptions.BoundedCapacity = _bufferSize;
var actionBlock = new ActionBlock <DomainEvent>((Action <DomainEvent>)dispatcher.DispatchEvent, consumerOptions);
consumers.Add(actionBlock);
foreach (var projection in dispatcher.Projections)
{
_projectionInspector.InspectProjectionForEvents(projection.GetType());
}
}
var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList();
var _broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, 3000);
_buffer.LinkTo(_broadcaster, new DataflowLinkOptions()
{
PropagateCompletion = true
});
Task.Factory.StartNew(() => StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers));
}
MetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
return(_status);
}