public void SetUp()
{
var config = new DocumentStoreTestConfiguration();
MongoDbTestConnectionProvider.DropTestsTenant();
config.SetTestAddress(TestConfig.ServerAddress);
_documentStoreService = new DocumentStoreBootstrapper();
_documentStoreService.Start(config);
TenantContext.Enter(new TenantId(TestConfig.Tenant));
var tenant = ContainerAccessor.Instance.Resolve <TenantManager>().Current;
_bus = tenant.Container.Resolve <ICommandBus>();
_filestore = tenant.Container.Resolve <IBlobStore>();
Assert.IsTrue(_bus is IInProcessCommandBus);
//Issue: https://github.com/ProximoSrl/Jarvis.DocumentStore/issues/26
//you need to resolve the IReader that in turns resolves the ProjectionEngine, becauase if you
//directly resolve the ITriggerProjectionsUpdate, projection engine will be resolved multiple times.
tenant.Container.Resolve <IReader <StreamReadModel, Int64> >();
_projections = tenant.Container.Resolve <ITriggerProjectionsUpdate>();
_handleWriter = tenant.Container.Resolve <IDocumentWriter>();
_documentReader = tenant.Container.Resolve <IReader <DocumentDescriptorReadModel, DocumentDescriptorId> >();
}
public void resolving_missing_tenant_shuld_inject_null_tenant()
{
TenantContext.Enter(new TenantId("not_yet_registered_in_container"));
var service = _container.Resolve <Service>();
Assert.AreSame(NullTenant.Instance, service.Tenant);
}
public void resolving_registered_tenant_should_inject_correct_tenant()
{
var tenantId = new TenantId("tenant");
var settings = new TenantATestSettings();
_tenantManager.AddTenant(settings);
TenantContext.Enter(tenantId);
var service = _container.Resolve <Service>();
Assert.AreSame(_tenantManager.GetTenant(tenantId), service.Tenant);
}
private void Init()
{
_maxDispatchedCheckpoint = 0;
DumpProjections();
TenantContext.Enter(_config.TenantId);
_housekeeper.Init();
_eventstore = Wireup
.Init()
.LogTo(t => new NEventStoreLog4NetLogger(LoggerFactory.Create(t)))
.UsingMongoPersistence(() => _config.EventStoreConnectionString, new DocumentObjectSerializer())
.InitializeStorageEngine()
.Build();
ConfigureProjections();
// cleanup
_housekeeper.RemoveAll(_eventstore.Advanced);
var allSlots = _projectionsBySlot.Keys.ToArray();
var allClients = new List <ICommitPollingClient> ();
//recreate all polling clients.
foreach (var bucket in _config.BucketInfo)
{
var client = _pollingClientFactory.Create(_eventstore.Advanced, "bucket: " + String.Join(",", bucket.Slots));
allClients.Add(client);
_bucketToClient.Add(bucket, client);
}
_clients = allClients.ToArray();
foreach (var slotName in allSlots)
{
MetricsHelper.CreateMeterForDispatcherCountSlot(slotName);
var startCheckpoint = GetStartCheckpointForSlot(slotName);
Logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint);
var name = slotName;
//find right consumer
var slotBucket = _config.BucketInfo.SingleOrDefault(b =>
b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ??
_config.BucketInfo.Single(b => b.Slots[0] == "*");
var client = _bucketToClient[slotBucket];
client.AddConsumer(commit => DispatchCommit(commit, name, startCheckpoint));
}
MetricsHelper.SetProjectionEngineCurrentDispatchCount(() => _countOfConcurrentDispatchingCommit);
}
public void disposing_disposable_service_should_not_dispose_tenant()
{
var tenant_a = new TenantId("a");
_tenantManager.AddTenant(new TenantATestSettings());
var tenant = _tenantManager.GetTenant(tenant_a);
TenantContext.Enter(tenant_a);
using (var service = _container.Resolve <DisposableService>())
{
Assert.AreSame(tenant, service.Tenant);
}
Assert.IsFalse(tenant.HasBeedDisposed);
}
private async Task InitAsync()
{
_maxDispatchedCheckpoint = 0;
DumpProjections();
TenantContext.Enter(_config.TenantId);
await _housekeeper.InitAsync().ConfigureAwait(false);
await ConfigureProjectionsAsync().ConfigureAwait(false);
// cleanup
await _housekeeper.RemoveAllAsync(_persistence).ConfigureAwait(false);
var allSlots = _projectionsBySlot.Keys.ToArray();
var allClients = new List <ICommitPollingClient>();
//recreate all polling clients.
foreach (var bucket in _config.BucketInfo)
{
string pollerId = "bucket: " + String.Join(",", bucket.Slots);
var client = _pollingClientFactory.Create(_persistence, pollerId);
allClients.Add(client);
_bucketToClient.Add(bucket, client);
client.Configure(GetStartGlobalCheckpoint(bucket.Slots), 4000);
}
_clients = allClients.ToArray();
foreach (var slotName in allSlots)
{
KernelMetricsHelper.CreateMeterForDispatcherCountSlot(slotName);
var startCheckpoint = GetStartCheckpointForSlot(slotName);
_logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint);
var name = slotName;
//find right consumer
var slotBucket = _config.BucketInfo.SingleOrDefault(b =>
b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ??
_config.BucketInfo.Single(b => b.Slots[0] == "*");
var client = _bucketToClient[slotBucket];
client.AddConsumer($"SLOT: {slotName}", commit => DispatchCommitAsync(commit, name, startCheckpoint));
}
Initialized = true;
KernelMetricsHelper.SetProjectionEngineCurrentDispatchCount(() => _countOfConcurrentDispatchingCommit);
}
public void SetUp()
{
BsonClassMapHelper.Clear();
var config = new DocumentStoreTestConfiguration();
MongoDbTestConnectionProvider.DropTestsTenant();
config.SetTestAddress(TestConfig.TestHostServiceAddress);
_documentStoreService = new DocumentStoreBootstrapper();
_documentStoreService.Start(config);
TenantContext.Enter(new TenantId(TestConfig.Tenant));
var tenant = ContainerAccessor.Instance.Resolve <TenantManager>().Current;
_writer = tenant.Container.Resolve <IDocumentWriter>();
}
public void should_resolve_two_tenants()
{
var tenant_a = new TenantId("a");
var tenant_b = new TenantId("b");
_tenantManager.AddTenant(new TenantATestSettings());
_tenantManager.AddTenant(new TenantBTestSettings());
TenantContext.Enter(tenant_a);
var service_a = _container.Resolve <Service>();
TenantContext.Enter(tenant_b);
var service_b = _container.Resolve <Service>();
Assert.AreSame(_tenantManager.GetTenant(tenant_a), service_a.Tenant);
Assert.AreSame(_tenantManager.GetTenant(tenant_b), service_b.Tenant);
}
private void Init()
{
_maxDispatchedCheckpoint = 0;
DumpProjections();
_stopping = new ManualResetEventSlim(false);
TenantContext.Enter(_config.TenantId);
_housekeeper.Init();
_eventstore = Wireup
.Init()
.LogTo(t => new NEventStoreLog4NetLogger(LoggerFactory.Create(t)))
.UsingMongoPersistence(() => _config.EventStoreConnectionString, new DocumentObjectSerializer())
.InitializeStorageEngine()
.Build();
ConfigureProjections();
// cleanup
_housekeeper.RemoveAll(_eventstore.Advanced);
var allSlots = _projectionsBySlot.Keys.ToArray();
_client.Create(_eventstore.Advanced, _config.PollingMsInterval);
var subscriptions = new List <IDisposable>();
_observeCommits = _client.ObserveFrom(GetStartGlobalCheckpoint());
foreach (var slotName in allSlots)
{
MetricsHelper.CreateMeterForDispatcherCountSlot(slotName);
var startCheckpoint = GetStartCheckpointForSlot(slotName);
Logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint);
var name = slotName;
subscriptions.Add(
_observeCommits.Subscribe(commit => DispatchCommit(commit, name, LongCheckpoint.Parse(startCheckpoint)),
HandleError));
}
_subscriptions = subscriptions.ToArray();
}
protected override Task <HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var routeData = request.GetConfiguration().Routes.GetRouteData(request);
if (routeData != null && routeData.Values.ContainsKey("MS_SubRoutes"))
{
var route = ((IHttpRouteData[])routeData.Values["MS_SubRoutes"]).First();
if (route.Values.ContainsKey("tenantid"))
{
string tenant = route.Values["tenantid"].ToString();
_logger.DebugFormat("Request {0} -> Tenant {1}", request.RequestUri, tenant);
TenantContext.Enter(new TenantId(tenant));
}
}
var result = base.SendAsync(request, cancellationToken);
return(result);
}
internal void DispatchEvent(DomainEvent evt)
{
var chkpoint = evt.CheckpointToken;
if (chkpoint > LastCheckpointDispatched)
{
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("Discharded event {0} commit {1} because last checkpoint dispatched for slot {2} is {3}.", evt.CommitId, evt.CheckpointToken, SlotName, _maxCheckpointDispatched);
}
return;
}
Interlocked.Increment(ref RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
//Console.WriteLine("[{0:00}] - Slot {1}", Thread.CurrentThread.ManagedThreadId, slotName);
if (_logger.IsDebugEnabled)
{
_logger.ThreadProperties["commit"] = evt.CommitId;
_logger.DebugFormat("Dispatching checkpoit {0} on tenant {1}", evt.CheckpointToken, _config.TenantId);
}
TenantContext.Enter(_config.TenantId);
try
{
if (_logger.IsDebugEnabled)
{
_logger.ThreadProperties["evType"] = evt.GetType().Name;
_logger.ThreadProperties["evMsId"] = evt.MessageId;
_logger.ThreadProperties["evCheckpointToken"] = evt.CheckpointToken;
}
string eventName = evt.GetType().Name;
foreach (var projection in _projections)
{
var cname = projection.GetCommonName();
if (_logger.IsDebugEnabled)
{
_logger.ThreadProperties["prj"] = cname;
}
bool handled;
long ticks = 0;
try
{
//pay attention, stopwatch consumes time.
var sw = new Stopwatch();
sw.Start();
handled = projection.Handle(evt, true);
sw.Stop();
ticks = sw.ElapsedTicks;
MetricsHelper.IncrementProjectionCounterRebuild(cname, SlotName, eventName, ticks);
}
catch (Exception ex)
{
_logger.FatalFormat(ex, "[Slot: {3} Projection: {4}] Failed checkpoint: {0} StreamId: {1} Event Name: {2}",
evt.CheckpointToken,
evt.AggregateId,
eventName,
SlotName,
cname
);
throw;
}
_metrics.Inc(cname, eventName, ticks);
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("[{3}] [{4}] Handled checkpoint {0}: {1} > {2}",
evt.CheckpointToken,
evt.AggregateId,
eventName,
SlotName,
cname
);
}
if (_logger.IsDebugEnabled)
{
_logger.ThreadProperties["prj"] = null;
}
}
ClearLoggerThreadPropertiesForEventDispatchLoop();
}
catch (Exception ex)
{
_logger.ErrorFormat(ex, "Error dispathing commit id: {0}\nMessage: {1}\nError: {2}",
evt.CheckpointToken, evt, ex.Message);
ClearLoggerThreadPropertiesForEventDispatchLoop();
throw;
}
_lastCheckpointRebuilded = chkpoint;
if (_logger.IsDebugEnabled)
{
_logger.ThreadProperties["commit"] = null;
}
Interlocked.Decrement(ref RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
}
internal async Task DispatchEventAsync(UnwindedDomainEvent unwindedEvent)
{
if (unwindedEvent == UnwindedDomainEvent.LastEvent)
{
Finished = true;
_lastCheckpointRebuilded = LastCheckpointDispatched; //Set to zero metrics, we dispatched everything.
return;
}
var chkpoint = unwindedEvent.CheckpointToken;
if (chkpoint > LastCheckpointDispatched)
{
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("Discharded event {0} commit {1} because last checkpoint dispatched for slot {2} is {3}.", unwindedEvent.CommitId, unwindedEvent.CheckpointToken, SlotName, _maxCheckpointDispatched);
}
return;
}
Interlocked.Increment(ref RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
TenantContext.Enter(_config.TenantId);
try
{
string eventName = unwindedEvent.EventType;
foreach (var projection in _projections)
{
var cname = projection.Info.CommonName;
long elapsedticks;
try
{
QueryPerformanceCounter(out long ticks1);
await projection.HandleAsync(unwindedEvent.GetEvent(), true).ConfigureAwait(false);
QueryPerformanceCounter(out long ticks2);
elapsedticks = ticks2 - ticks1;
KernelMetricsHelper.IncrementProjectionCounterRebuild(cname, SlotName, eventName, elapsedticks);
}
catch (Exception ex)
{
_logger.FatalFormat(ex, "[Slot: {3} Projection: {4}] Failed checkpoint: {0} StreamId: {1} Event Name: {2}",
unwindedEvent.CheckpointToken,
unwindedEvent.PartitionId,
eventName,
SlotName,
cname
);
HealthChecks.RegisterHealthCheck($"RebuildDispatcher, slot {SlotName} - FailedCheckpoint {unwindedEvent.CheckpointToken}", () =>
HealthCheckResult.Unhealthy(ex)
);
throw;
}
_metrics.Inc(cname, eventName, elapsedticks);
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("[{3}] [{4}] Handled checkpoint {0}: {1} > {2}",
unwindedEvent.CheckpointToken,
unwindedEvent.PartitionId,
eventName,
SlotName,
cname
);
}
}
}
catch (Exception ex)
{
_logger.ErrorFormat(ex, "Error dispathing commit id: {0}\nMessage: {1}\nError: {2}",
unwindedEvent.CheckpointToken, unwindedEvent.Event, ex.Message);
HealthChecks.RegisterHealthCheck($"RebuildDispatcher, slot {SlotName} - GeneralError", () =>
HealthCheckResult.Unhealthy(ex)
);
throw;
}
_lastCheckpointRebuilded = chkpoint;
KernelMetricsHelper.MarkEventInRebuildDispatchedCount(SlotName, 1);
Interlocked.Decrement(ref RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
}
internal void UploadFile(String jobFile, DocumentImportTask task)
{
String fname = "";
try
{
TenantContext.Enter(task.Tenant);
if (!task.Uri.IsFile)
{
LogAndThrow("Error importing task file {0}: Uri is not a file: {1}", jobFile, task.Uri);
}
fname = task.Uri.LocalPath;
if (FileHasImportFailureMarker(fname, task.FileTimestamp))
{
return;
}
if (!File.Exists(fname))
{
LogAndThrow("Error importing task file {0}: File missing: {1}", jobFile, fname);
}
var blobStore = GetBlobStoreForTenant();
var identityGenerator = GetIdentityGeneratorForTenant();
if (blobStore == null || identityGenerator == null)
{
Logger.ErrorFormat("Tenant {1} not found or not configured for file: {1}", task.Tenant, fname);
return;
}
BlobId blobId;
if (!String.IsNullOrEmpty(task.FileName))
{
//use the real file name from the task not the name of the file
using (FileStream fs = File.Open(fname, FileMode.Open, FileAccess.Read))
{
blobId = blobStore.Upload(task.Format, new FileNameWithExtension(task.FileName), fs);
}
}
else
{
//No filename given in task, use name of the blob
blobId = blobStore.Upload(task.Format, fname);
}
if (task.Format == OriginalFormat)
{
var descriptor = blobStore.GetDescriptor(blobId);
var fileName = new FileNameWithExtension(task.FileName);
var handleInfo = new DocumentHandleInfo(task.Handle, fileName, task.CustomData);
var documentId = identityGenerator.New <DocumentDescriptorId>();
var createDocument = new InitializeDocumentDescriptor(
documentId,
blobId,
handleInfo,
descriptor.Hash,
fileName
);
_commandBus.Send(createDocument, "import-from-file");
}
else
{
var reader = _tenantAccessor.Current.Container.Resolve <IDocumentWriter>();
var handle = reader.FindOneById(task.Handle);
var documentId = handle.DocumentDescriptorId;
var command = new AddFormatToDocumentDescriptor(
documentId,
task.Format,
blobId,
new PipelineId("user-content")
);
_commandBus.Send(command, "import-from-file");
}
TaskExecuted(task);
DeleteImportFailure(fname);
}
catch (Exception ex)
{
Logger.ErrorFormat(ex, "Job Import Queue - Error importing {0} - {1}", jobFile, ex.Message);
ImportFailure failure = new ImportFailure()
{
Error = ex.ToString(),
FileName = fname,
Timestamp = DateTime.Now,
ImportFileTimestampTicks = task.FileTimestamp.Ticks,
};
MarkImportFailure(failure);
}
finally
{
TenantContext.Exit();
}
}
/// <summary>
/// Start rebuild process.
/// </summary>
/// <returns></returns>
public async Task <RebuildStatus> RebuildAsync()
{
if (Logger.IsInfoEnabled)
{
Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId);
}
DumpProjections();
await _eventUnwinder.UnwindAsync().ConfigureAwait(false);
_status = new RebuildStatus();
TenantContext.Enter(_config.TenantId);
await ConfigureProjections().ConfigureAwait(false);
var allSlots = _projectionsBySlot.Keys.ToArray();
//initialize dispatching of the commits
foreach (var bucket in _config.BucketInfo)
{
_consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>());
_status.AddBucket();
}
//Setup the slots
foreach (var slotName in allSlots)
{
Logger.InfoFormat("Slot {0} will be rebuilded", slotName);
var projectionsForThisSlot = _projectionsBySlot[slotName];
Int64 maximumDispatchedValue = projectionsForThisSlot
.Select(p => _checkpointTracker.GetCheckpoint(p))
.Max();
var dispatcher = new RebuildProjectionSlotDispatcher(Logger, slotName, _config, projectionsForThisSlot, maximumDispatchedValue, _loggerThreadContextManager);
KernelMetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch);
_rebuildDispatchers.Add(dispatcher);
//find right consumer
var slotBucket = _config.BucketInfo.SingleOrDefault(b =>
b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ??
_config.BucketInfo.Single(b => b.Slots[0] == "*");
var consumerList = _consumers[slotBucket];
consumerList.Add(dispatcher);
}
//Creates TPL chain and start polling everything on the consumer
foreach (var consumer in _consumers)
{
var bucketInfo = String.Join(",", consumer.Key.Slots);
if (consumer.Value.Count == 0)
{
Logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo);
_status.BucketDone(bucketInfo, 0, 0, 0);
continue;
}
var consumerBufferOptions = new DataflowBlockOptions();
consumerBufferOptions.BoundedCapacity = consumer.Key.BufferSize;
var _buffer = new BufferBlock <UnwindedDomainEvent>(consumerBufferOptions);
ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions();
executionOption.BoundedCapacity = consumer.Key.BufferSize;
var dispatcherList = consumer.Value;
_projectionInspector.ResetHandledEvents();
List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> > consumers =
new List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> >();
foreach (var dispatcher in dispatcherList)
{
ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions();
consumerOptions.BoundedCapacity = consumer.Key.BufferSize;
var actionBlock = new ActionBlock <UnwindedDomainEvent>((Func <UnwindedDomainEvent, Task>)dispatcher.DispatchEventAsync, consumerOptions);
HashSet <Type> eventsOfThisSlot = new HashSet <Type>();
foreach (var projection in dispatcher.Projections)
{
var domainEventTypesHandledByThisProjection = _projectionInspector.InspectProjectionForEvents(projection.GetType());
foreach (var type in domainEventTypesHandledByThisProjection)
{
eventsOfThisSlot.Add(type);
}
}
SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> slotInfo =
new SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent>(
actionBlock,
dispatcher.SlotName,
eventsOfThisSlot);
consumers.Add(slotInfo);
KernelMetricsHelper.CreateMeterForRebuildDispatcherBuffer(dispatcher.SlotName, () => actionBlock.InputCount);
}
var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList();
var broadcaster = SlotGuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, consumer.Key.BufferSize);
_buffer.LinkTo(broadcaster, new DataflowLinkOptions()
{
PropagateCompletion = true
});
KernelMetricsHelper.CreateGaugeForRebuildFirstBuffer(bucketInfo, () => _buffer.Count);
KernelMetricsHelper.CreateGaugeForRebuildBucketDBroadcasterBuffer(bucketInfo, () => broadcaster.InputCount);
KernelMetricsHelper.CreateMeterForRebuildEventCompleted(bucketInfo);
//fire each bucket in own thread
#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
Task.Factory.StartNew(() => StartPoll(
_buffer,
broadcaster,
bucketInfo,
dispatcherList,
allTypeHandledStringList,
consumers));
//await StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers).ConfigureAwait(false);
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
}
KernelMetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
return(_status);
}
private async Task DispatchCommitAsync(IChunk chunk, string slotName, Int64 startCheckpoint)
{
Interlocked.Increment(ref _countOfConcurrentDispatchingCommit);
_loggerThreadContextManager.SetContextProperty("commit", $"{chunk.OperationId}/{chunk.Position}");
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("Dispatching checkpoit {0} on tenant {1}", chunk.Position, _config.TenantId);
}
TenantContext.Enter(_config.TenantId);
var chkpoint = chunk.Position;
if (!lastCheckpointDispatched.ContainsKey(slotName))
{
lastCheckpointDispatched[slotName] = 0;
}
if (lastCheckpointDispatched[slotName] >= chkpoint)
{
var error = String.Format("Sequence broken, last checkpoint for slot {0} was {1} and now we dispatched {2}",
slotName, lastCheckpointDispatched[slotName], chkpoint);
_logger.Error(error);
throw new JarvisFrameworkEngineException(error);
}
if (lastCheckpointDispatched[slotName] + 1 != chkpoint && lastCheckpointDispatched[slotName] > 0)
{
_logger.DebugFormat("Sequence of commit not consecutive (probably holes), last dispatched {0} receiving {1}",
lastCheckpointDispatched[slotName], chkpoint);
}
lastCheckpointDispatched[slotName] = chkpoint;
if (chkpoint <= startCheckpoint)
{
//Already dispatched, skip it.
Interlocked.Decrement(ref _countOfConcurrentDispatchingCommit);
return;
}
var projections = _projectionsBySlot[slotName];
object[] events = GetArrayOfObjectFromChunk(chunk);
var eventCount = events.Length;
//now it is time to dispatch the commit, we will dispatch each projection
//and for each projection we dispatch all the events present in changeset
bool someProjectionProcessedTheEvent = false;
foreach (var projection in projections)
{
var cname = projection.Info.CommonName;
Object eventMessage = null;
try
{
//Foreach projection we need to dispach every event
for (int index = 0; index < eventCount; index++)
{
eventMessage = events[index];
var message = eventMessage as IMessage;
string eventName = eventMessage.GetType().Name;
_loggerThreadContextManager.SetContextProperty("evType", eventName);
_loggerThreadContextManager.SetContextProperty("evMsId", message?.MessageId);
_loggerThreadContextManager.SetContextProperty("evCheckpointToken", chunk.Position);
_loggerThreadContextManager.SetContextProperty("prj", cname);
var checkpointStatus = _checkpointTracker.GetCheckpointStatus(cname, chunk.Position);
long ticks = 0;
try
{
//pay attention, stopwatch consumes time.
var sw = new Stopwatch();
sw.Start();
var eventProcessed = await projection
.HandleAsync(eventMessage, checkpointStatus.IsRebuilding)
.ConfigureAwait(false);
someProjectionProcessedTheEvent |= eventProcessed;
sw.Stop();
ticks = sw.ElapsedTicks;
KernelMetricsHelper.IncrementProjectionCounter(cname, slotName, eventName, ticks, sw.ElapsedMilliseconds);
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("[Slot:{3}] [Projection {4}] Handled checkpoint {0}: {1} > {2}",
chunk.Position,
chunk.PartitionId,
$"eventName: {eventName} [event N°{index}]",
slotName,
cname
);
}
}
catch (Exception ex)
{
var error = String.Format("[Slot: {3} Projection: {4}] Failed checkpoint: {0} StreamId: {1} Event Name: {2}",
chunk.Position,
chunk.PartitionId,
eventName,
slotName,
cname);
_logger.Fatal(error, ex);
_engineFatalErrors.Add(String.Format("{0}\n{1}", error, ex));
throw;
}
} //End of event cycle
}
catch (Exception ex)
{
_loggerThreadContextManager.ClearContextProperty("commit");
_logger.ErrorFormat(ex, "Error dispathing Chunk [{0}]\n Message: {1}\n Error: {2}\n",
chunk.Position, eventMessage?.GetType()?.Name, ex.Message);
throw;
}
finally
{
_loggerThreadContextManager.ClearContextProperty("evType");
_loggerThreadContextManager.ClearContextProperty("evMsId");
_loggerThreadContextManager.ClearContextProperty("evCheckpointToken");
_loggerThreadContextManager.ClearContextProperty("prj");
}
projection.CheckpointProjected(chunk.Position);
//TODO: Evaluate if it is needed to update single projection checkpoint
//Update this projection, set all events of this checkpoint as dispatched.
//await _checkpointTracker.UpdateProjectionCheckpointAsync(cname, chunk.Position).ConfigureAwait(false);
} //End of projection cycle.
await _checkpointTracker.UpdateSlotAndSetCheckpointAsync(
slotName,
projections.Select(_ => _.Info.CommonName),
chunk.Position,
someEventDispatched : someProjectionProcessedTheEvent).ConfigureAwait(false);
KernelMetricsHelper.MarkCommitDispatchedCount(slotName, 1);
await _notifyCommitHandled.SetDispatched(slotName, chunk).ConfigureAwait(false);
// ok in multithread wihout locks!
if (_maxDispatchedCheckpoint < chkpoint)
{
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("Updating last dispatched checkpoint from {0} to {1}",
_maxDispatchedCheckpoint,
chkpoint
);
}
_maxDispatchedCheckpoint = chkpoint;
}
_loggerThreadContextManager.ClearContextProperty("commit");
Interlocked.Decrement(ref _countOfConcurrentDispatchingCommit);
}
void DispatchCommit(ICommit commit, string slotName, Int64 startCheckpoint)
{
Interlocked.Increment(ref _countOfConcurrentDispatchingCommit);
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["commit"] = commit.CommitId;
}
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Dispatching checkpoit {0} on tenant {1}", commit.CheckpointToken, _config.TenantId);
}
TenantContext.Enter(_config.TenantId);
var chkpoint = commit.CheckpointToken;
if (!lastCheckpointDispatched.ContainsKey(slotName))
{
lastCheckpointDispatched[slotName] = 0;
}
if (lastCheckpointDispatched[slotName] >= chkpoint)
{
var error = String.Format("Sequence broken, last checkpoint for slot {0} was {1} and now we dispatched {2}",
slotName, lastCheckpointDispatched[slotName], chkpoint);
Logger.Error(error);
throw new ApplicationException(error);
}
if (lastCheckpointDispatched[slotName] + 1 != chkpoint)
{
if (lastCheckpointDispatched[slotName] > 0)
{
Logger.DebugFormat("Sequence of commit non consecutive, last dispatched {0} receiving {1}",
lastCheckpointDispatched[slotName], chkpoint);
}
}
lastCheckpointDispatched[slotName] = chkpoint;
if (chkpoint <= startCheckpoint)
{
//Already dispatched, skip it.
Interlocked.Decrement(ref _countOfConcurrentDispatchingCommit);
return;
}
var projections = _projectionsBySlot[slotName];
bool dispatchCommit = false;
var eventCount = commit.Events.Count;
var projectionToUpdate = new List <string>();
for (int index = 0; index < eventCount; index++)
{
var eventMessage = commit.Events.ElementAt(index);
try
{
var evt = (DomainEvent)eventMessage.Body;
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["evType"] = evt.GetType().Name;
}
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["evMsId"] = evt.MessageId;
}
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["evCheckpointToken"] = commit.CheckpointToken;
}
string eventName = evt.GetType().Name;
foreach (var projection in projections)
{
var cname = projection.GetCommonName();
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["prj"] = cname;
}
var checkpointStatus = _checkpointTracker.GetCheckpointStatus(cname, commit.CheckpointToken);
bool handled;
long ticks = 0;
try
{
//pay attention, stopwatch consumes time.
var sw = new Stopwatch();
sw.Start();
handled = projection.Handle(evt, checkpointStatus.IsRebuilding);
sw.Stop();
ticks = sw.ElapsedTicks;
MetricsHelper.IncrementProjectionCounter(cname, slotName, eventName, ticks);
}
catch (Exception ex)
{
var error = String.Format("[Slot: {3} Projection: {4}] Failed checkpoint: {0} StreamId: {1} Event Name: {2}",
commit.CheckpointToken,
commit.StreamId,
eventName,
slotName,
cname);
Logger.Fatal(error, ex);
_engineFatalErrors.Add(error);
throw;
}
if (handled)
{
_metrics.Inc(cname, eventName, ticks);
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("[{3}] [{4}] Handled checkpoint {0}: {1} > {2}",
commit.CheckpointToken,
commit.StreamId,
eventName,
slotName,
cname
);
}
}
if (!checkpointStatus.IsRebuilding)
{
if (handled)
{
dispatchCommit = true;
}
projectionToUpdate.Add(cname);
}
else
{
if (checkpointStatus.IsLast && (index == eventCount - 1))
{
projection.StopRebuild();
var meter = _metrics.GetMeter(cname);
_checkpointTracker.RebuildEnded(projection, meter);
Logger.InfoFormat("Rebuild done {0} @ {1}",
projection.GetType().FullName,
commit.CheckpointToken
);
}
}
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["prj"] = null;
}
}
ClearLoggerThreadPropertiesForEventDispatchLoop();
}
catch (Exception ex)
{
Logger.ErrorFormat(ex, "Error dispathing commit id: {0}\nMessage: {1}\nError: {2}",
commit.CheckpointToken, eventMessage.Body, ex.Message);
ClearLoggerThreadPropertiesForEventDispatchLoop();
throw;
}
}
if (projectionToUpdate.Count == 0)
{
//I'm in rebuilding or no projection had run any events, only update slot
_checkpointTracker.UpdateSlot(slotName, commit.CheckpointToken);
}
else
{
//I'm not on rebuilding, we have projection to update, update everything with one call.
_checkpointTracker.UpdateSlotAndSetCheckpoint(slotName, projectionToUpdate, commit.CheckpointToken);
}
MetricsHelper.MarkCommitDispatchedCount(slotName, 1);
if (dispatchCommit)
{
_notifyCommitHandled.SetDispatched(commit);
}
// ok in multithread wihout locks!
if (_maxDispatchedCheckpoint < chkpoint)
{
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Updating last dispatched checkpoint from {0} to {1}",
_maxDispatchedCheckpoint,
chkpoint
);
}
_maxDispatchedCheckpoint = chkpoint;
}
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["commit"] = null;
}
Interlocked.Decrement(ref _countOfConcurrentDispatchingCommit);
}
void DispatchCommit(ICommit commit, string slotName, LongCheckpoint startCheckpoint)
{
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["commit"] = commit.CommitId;
}
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Dispatching checkpoit {0} on tenant {1}", commit.CheckpointToken, _config.TenantId);
}
TenantContext.Enter(_config.TenantId);
var chkpoint = LongCheckpoint.Parse(commit.CheckpointToken);
if (chkpoint.LongValue <= startCheckpoint.LongValue)
{
// già fatta
return;
}
var projections = _projectionsBySlot[slotName];
bool dispatchCommit = false;
var eventMessages = commit.Events.ToArray();
var projectionToUpdate = new List <string>();
for (int index = 0; index < eventMessages.Length; index++)
{
var eventMessage = eventMessages[index];
try
{
var evt = (DomainEvent)eventMessage.Body;
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["evType"] = evt.GetType().Name;
}
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["evMsId"] = evt.MessageId;
}
string eventName = evt.GetType().Name;
foreach (var projection in projections)
{
var cname = projection.GetCommonName();
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["prj"] = cname;
}
var checkpointStatus = _checkpointTracker.GetCheckpointStatus(cname, commit.CheckpointToken);
bool handled;
long ticks = 0;
try
{
var sw = new Stopwatch();
sw.Start();
//if (Logger.IsDebugEnabled)Logger.DebugFormat("Handling {0} with {1}", commit.CheckpointToken, projection.GetType().Name);
handled = projection.Handle(evt, checkpointStatus.IsRebuilding);
//if (Logger.IsDebugEnabled)Logger.DebugFormat("Handled {0} with {1}", commit.CheckpointToken, projection.GetType().Name);
sw.Stop();
ticks = sw.ElapsedTicks;
}
catch (Exception ex)
{
Logger.FatalFormat(ex, "[{3}] Failed checkpoint {0}: {1} > {2}",
commit.CheckpointToken,
commit.StreamId,
eventName,
slotName
);
throw;
}
if (handled)
{
_metrics.Inc(cname, eventName, ticks);
Logger.DebugFormat("[{3}] [{4}] Handled checkpoint {0}: {1} > {2}",
commit.CheckpointToken,
commit.StreamId,
eventName,
slotName,
cname
);
}
if (!checkpointStatus.IsRebuilding)
{
if (handled)
{
dispatchCommit = true;
}
projectionToUpdate.Add(cname);
}
else
{
if (checkpointStatus.IsLast && (index == eventMessages.Length - 1))
{
projection.StopRebuild();
var meter = _metrics.GetMeter(cname);
_checkpointTracker.RebuildEnded(projection, meter);
Logger.InfoFormat("Rebuild done {0} @ {1}",
projection.GetType().FullName,
commit.CheckpointToken
);
}
}
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["prj"] = null;
}
}
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["evType"] = null;
}
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["evMsId"] = null;
}
}
catch (Exception ex)
{
if (Logger.IsDebugEnabled)
{
Logger.ErrorFormat(ex, "{0} -> {1}", eventMessage.Body, ex.Message);
}
throw;
}
}
_checkpointTracker.UpdateSlot(slotName, commit.CheckpointToken);
MetricsHelper.MarkCommitDispatchedCount(slotName, 1);
foreach (var cname in projectionToUpdate)
{
_checkpointTracker.SetCheckpoint(cname, commit.CheckpointToken);
}
if (dispatchCommit)
{
_notifyCommitHandled.SetDispatched(commit);
}
// ok in multithread wihout locks!
if (_maxDispatchedCheckpoint < chkpoint.LongValue)
{
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Updating last dispatched checkpoint from {0} to {1}",
_maxDispatchedCheckpoint,
chkpoint.LongValue
);
}
_maxDispatchedCheckpoint = chkpoint.LongValue;
}
if (Logger.IsDebugEnabled)
{
Logger.ThreadProperties["commit"] = null;
}
}
public RebuildStatus Rebuild()
{
if (Logger.IsInfoEnabled)
{
Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId);
}
DumpProjections();
_eventUnwinder.Unwind();
_status = new RebuildStatus();
TenantContext.Enter(_config.TenantId);
ConfigureProjections();
var allSlots = _projectionsBySlot.Keys.ToArray();
//initialize dispatching of the commits
foreach (var bucket in _config.BucketInfo)
{
_consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>());
_status.AddBucket();
}
//Setup the slots
foreach (var slotName in allSlots)
{
var startCheckpoint = GetStartCheckpointForSlot(slotName);
Logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint);
var projectionsForThisSlot = _projectionsBySlot[slotName];
Int64 maximumDispatchedValue = projectionsForThisSlot
.Select(p => _checkpointTracker.GetCheckpoint(p))
.Max();
var dispatcher = new RebuildProjectionSlotDispatcher(_logger, slotName, _config, projectionsForThisSlot, _checkpointTracker, maximumDispatchedValue);
MetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch);
_rebuildDispatchers.Add(dispatcher);
//find right consumer
var slotBucket = _config.BucketInfo.SingleOrDefault(b =>
b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ??
_config.BucketInfo.Single(b => b.Slots[0] == "*");
var consumerList = _consumers[slotBucket];
consumerList.Add(dispatcher);
}
//now start tpl and start polling in other threads.
foreach (var consumer in _consumers)
{
var bucketInfo = String.Join(",", consumer.Key.Slots);
if (consumer.Value.Count == 0)
{
_logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo);
_status.BucketDone(bucketInfo, 0, 0, 0);
continue;
}
DataflowBlockOptions consumerBufferOptions = new DataflowBlockOptions();
consumerBufferOptions.BoundedCapacity = _bufferSize;
var _buffer = new BufferBlock <DomainEvent>(consumerBufferOptions);
ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions();
executionOption.BoundedCapacity = _bufferSize;
var dispatcherList = consumer.Value;
_projectionInspector.ResetHandledEvents();
List <ActionBlock <DomainEvent> > consumers = new List <ActionBlock <DomainEvent> >();
foreach (var dispatcher in dispatcherList)
{
ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions();
consumerOptions.BoundedCapacity = _bufferSize;
var actionBlock = new ActionBlock <DomainEvent>((Action <DomainEvent>)dispatcher.DispatchEvent, consumerOptions);
consumers.Add(actionBlock);
foreach (var projection in dispatcher.Projections)
{
_projectionInspector.InspectProjectionForEvents(projection.GetType());
}
}
var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList();
var _broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, 3000);
_buffer.LinkTo(_broadcaster, new DataflowLinkOptions()
{
PropagateCompletion = true
});
Task.Factory.StartNew(() => StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers));
}
MetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
return(_status);
}
