public void SetUp() { var config = new DocumentStoreTestConfiguration(); MongoDbTestConnectionProvider.DropTestsTenant(); config.SetTestAddress(TestConfig.ServerAddress); _documentStoreService = new DocumentStoreBootstrapper(); _documentStoreService.Start(config); TenantContext.Enter(new TenantId(TestConfig.Tenant)); var tenant = ContainerAccessor.Instance.Resolve <TenantManager>().Current; _bus = tenant.Container.Resolve <ICommandBus>(); _filestore = tenant.Container.Resolve <IBlobStore>(); Assert.IsTrue(_bus is IInProcessCommandBus); //Issue: https://github.com/ProximoSrl/Jarvis.DocumentStore/issues/26 //you need to resolve the IReader that in turns resolves the ProjectionEngine, becauase if you //directly resolve the ITriggerProjectionsUpdate, projection engine will be resolved multiple times. tenant.Container.Resolve <IReader <StreamReadModel, Int64> >(); _projections = tenant.Container.Resolve <ITriggerProjectionsUpdate>(); _handleWriter = tenant.Container.Resolve <IDocumentWriter>(); _documentReader = tenant.Container.Resolve <IReader <DocumentDescriptorReadModel, DocumentDescriptorId> >(); }
public void resolving_missing_tenant_shuld_inject_null_tenant() { TenantContext.Enter(new TenantId("not_yet_registered_in_container")); var service = _container.Resolve <Service>(); Assert.AreSame(NullTenant.Instance, service.Tenant); }
public void resolving_registered_tenant_should_inject_correct_tenant() { var tenantId = new TenantId("tenant"); var settings = new TenantATestSettings(); _tenantManager.AddTenant(settings); TenantContext.Enter(tenantId); var service = _container.Resolve <Service>(); Assert.AreSame(_tenantManager.GetTenant(tenantId), service.Tenant); }
private void Init() { _maxDispatchedCheckpoint = 0; DumpProjections(); TenantContext.Enter(_config.TenantId); _housekeeper.Init(); _eventstore = Wireup .Init() .LogTo(t => new NEventStoreLog4NetLogger(LoggerFactory.Create(t))) .UsingMongoPersistence(() => _config.EventStoreConnectionString, new DocumentObjectSerializer()) .InitializeStorageEngine() .Build(); ConfigureProjections(); // cleanup _housekeeper.RemoveAll(_eventstore.Advanced); var allSlots = _projectionsBySlot.Keys.ToArray(); var allClients = new List <ICommitPollingClient> (); //recreate all polling clients. foreach (var bucket in _config.BucketInfo) { var client = _pollingClientFactory.Create(_eventstore.Advanced, "bucket: " + String.Join(",", bucket.Slots)); allClients.Add(client); _bucketToClient.Add(bucket, client); } _clients = allClients.ToArray(); foreach (var slotName in allSlots) { MetricsHelper.CreateMeterForDispatcherCountSlot(slotName); var startCheckpoint = GetStartCheckpointForSlot(slotName); Logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint); var name = slotName; //find right consumer var slotBucket = _config.BucketInfo.SingleOrDefault(b => b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ?? _config.BucketInfo.Single(b => b.Slots[0] == "*"); var client = _bucketToClient[slotBucket]; client.AddConsumer(commit => DispatchCommit(commit, name, startCheckpoint)); } MetricsHelper.SetProjectionEngineCurrentDispatchCount(() => _countOfConcurrentDispatchingCommit); }
public void disposing_disposable_service_should_not_dispose_tenant() { var tenant_a = new TenantId("a"); _tenantManager.AddTenant(new TenantATestSettings()); var tenant = _tenantManager.GetTenant(tenant_a); TenantContext.Enter(tenant_a); using (var service = _container.Resolve <DisposableService>()) { Assert.AreSame(tenant, service.Tenant); } Assert.IsFalse(tenant.HasBeedDisposed); }
private async Task InitAsync() { _maxDispatchedCheckpoint = 0; DumpProjections(); TenantContext.Enter(_config.TenantId); await _housekeeper.InitAsync().ConfigureAwait(false); await ConfigureProjectionsAsync().ConfigureAwait(false); // cleanup await _housekeeper.RemoveAllAsync(_persistence).ConfigureAwait(false); var allSlots = _projectionsBySlot.Keys.ToArray(); var allClients = new List <ICommitPollingClient>(); //recreate all polling clients. foreach (var bucket in _config.BucketInfo) { string pollerId = "bucket: " + String.Join(",", bucket.Slots); var client = _pollingClientFactory.Create(_persistence, pollerId); allClients.Add(client); _bucketToClient.Add(bucket, client); client.Configure(GetStartGlobalCheckpoint(bucket.Slots), 4000); } _clients = allClients.ToArray(); foreach (var slotName in allSlots) { KernelMetricsHelper.CreateMeterForDispatcherCountSlot(slotName); var startCheckpoint = GetStartCheckpointForSlot(slotName); _logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint); var name = slotName; //find right consumer var slotBucket = _config.BucketInfo.SingleOrDefault(b => b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ?? _config.BucketInfo.Single(b => b.Slots[0] == "*"); var client = _bucketToClient[slotBucket]; client.AddConsumer($"SLOT: {slotName}", commit => DispatchCommitAsync(commit, name, startCheckpoint)); } Initialized = true; KernelMetricsHelper.SetProjectionEngineCurrentDispatchCount(() => _countOfConcurrentDispatchingCommit); }
public void SetUp() { BsonClassMapHelper.Clear(); var config = new DocumentStoreTestConfiguration(); MongoDbTestConnectionProvider.DropTestsTenant(); config.SetTestAddress(TestConfig.TestHostServiceAddress); _documentStoreService = new DocumentStoreBootstrapper(); _documentStoreService.Start(config); TenantContext.Enter(new TenantId(TestConfig.Tenant)); var tenant = ContainerAccessor.Instance.Resolve <TenantManager>().Current; _writer = tenant.Container.Resolve <IDocumentWriter>(); }
public void should_resolve_two_tenants() { var tenant_a = new TenantId("a"); var tenant_b = new TenantId("b"); _tenantManager.AddTenant(new TenantATestSettings()); _tenantManager.AddTenant(new TenantBTestSettings()); TenantContext.Enter(tenant_a); var service_a = _container.Resolve <Service>(); TenantContext.Enter(tenant_b); var service_b = _container.Resolve <Service>(); Assert.AreSame(_tenantManager.GetTenant(tenant_a), service_a.Tenant); Assert.AreSame(_tenantManager.GetTenant(tenant_b), service_b.Tenant); }
private void Init() { _maxDispatchedCheckpoint = 0; DumpProjections(); _stopping = new ManualResetEventSlim(false); TenantContext.Enter(_config.TenantId); _housekeeper.Init(); _eventstore = Wireup .Init() .LogTo(t => new NEventStoreLog4NetLogger(LoggerFactory.Create(t))) .UsingMongoPersistence(() => _config.EventStoreConnectionString, new DocumentObjectSerializer()) .InitializeStorageEngine() .Build(); ConfigureProjections(); // cleanup _housekeeper.RemoveAll(_eventstore.Advanced); var allSlots = _projectionsBySlot.Keys.ToArray(); _client.Create(_eventstore.Advanced, _config.PollingMsInterval); var subscriptions = new List <IDisposable>(); _observeCommits = _client.ObserveFrom(GetStartGlobalCheckpoint()); foreach (var slotName in allSlots) { MetricsHelper.CreateMeterForDispatcherCountSlot(slotName); var startCheckpoint = GetStartCheckpointForSlot(slotName); Logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint); var name = slotName; subscriptions.Add( _observeCommits.Subscribe(commit => DispatchCommit(commit, name, LongCheckpoint.Parse(startCheckpoint)), HandleError)); } _subscriptions = subscriptions.ToArray(); }
protected override Task <HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { var routeData = request.GetConfiguration().Routes.GetRouteData(request); if (routeData != null && routeData.Values.ContainsKey("MS_SubRoutes")) { var route = ((IHttpRouteData[])routeData.Values["MS_SubRoutes"]).First(); if (route.Values.ContainsKey("tenantid")) { string tenant = route.Values["tenantid"].ToString(); _logger.DebugFormat("Request {0} -> Tenant {1}", request.RequestUri, tenant); TenantContext.Enter(new TenantId(tenant)); } } var result = base.SendAsync(request, cancellationToken); return(result); }
internal void DispatchEvent(DomainEvent evt) { var chkpoint = evt.CheckpointToken; if (chkpoint > LastCheckpointDispatched) { if (_logger.IsDebugEnabled) { _logger.DebugFormat("Discharded event {0} commit {1} because last checkpoint dispatched for slot {2} is {3}.", evt.CommitId, evt.CheckpointToken, SlotName, _maxCheckpointDispatched); } return; } Interlocked.Increment(ref RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit); //Console.WriteLine("[{0:00}] - Slot {1}", Thread.CurrentThread.ManagedThreadId, slotName); if (_logger.IsDebugEnabled) { _logger.ThreadProperties["commit"] = evt.CommitId; _logger.DebugFormat("Dispatching checkpoit {0} on tenant {1}", evt.CheckpointToken, _config.TenantId); } TenantContext.Enter(_config.TenantId); try { if (_logger.IsDebugEnabled) { _logger.ThreadProperties["evType"] = evt.GetType().Name; _logger.ThreadProperties["evMsId"] = evt.MessageId; _logger.ThreadProperties["evCheckpointToken"] = evt.CheckpointToken; } string eventName = evt.GetType().Name; foreach (var projection in _projections) { var cname = projection.GetCommonName(); if (_logger.IsDebugEnabled) { _logger.ThreadProperties["prj"] = cname; } bool handled; long ticks = 0; try { //pay attention, stopwatch consumes time. var sw = new Stopwatch(); sw.Start(); handled = projection.Handle(evt, true); sw.Stop(); ticks = sw.ElapsedTicks; MetricsHelper.IncrementProjectionCounterRebuild(cname, SlotName, eventName, ticks); } catch (Exception ex) { _logger.FatalFormat(ex, "[Slot: {3} Projection: {4}] Failed checkpoint: {0} StreamId: {1} Event Name: {2}", evt.CheckpointToken, evt.AggregateId, eventName, SlotName, cname ); throw; } _metrics.Inc(cname, eventName, ticks); if (_logger.IsDebugEnabled) { _logger.DebugFormat("[{3}] [{4}] Handled checkpoint {0}: {1} > {2}", evt.CheckpointToken, evt.AggregateId, eventName, SlotName, cname ); } if (_logger.IsDebugEnabled) { _logger.ThreadProperties["prj"] = null; } } ClearLoggerThreadPropertiesForEventDispatchLoop(); } catch (Exception ex) { _logger.ErrorFormat(ex, "Error dispathing commit id: {0}\nMessage: {1}\nError: {2}", evt.CheckpointToken, evt, ex.Message); ClearLoggerThreadPropertiesForEventDispatchLoop(); throw; } _lastCheckpointRebuilded = chkpoint; if (_logger.IsDebugEnabled) { _logger.ThreadProperties["commit"] = null; } Interlocked.Decrement(ref RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit); }
internal async Task DispatchEventAsync(UnwindedDomainEvent unwindedEvent) { if (unwindedEvent == UnwindedDomainEvent.LastEvent) { Finished = true; _lastCheckpointRebuilded = LastCheckpointDispatched; //Set to zero metrics, we dispatched everything. return; } var chkpoint = unwindedEvent.CheckpointToken; if (chkpoint > LastCheckpointDispatched) { if (_logger.IsDebugEnabled) { _logger.DebugFormat("Discharded event {0} commit {1} because last checkpoint dispatched for slot {2} is {3}.", unwindedEvent.CommitId, unwindedEvent.CheckpointToken, SlotName, _maxCheckpointDispatched); } return; } Interlocked.Increment(ref RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit); TenantContext.Enter(_config.TenantId); try { string eventName = unwindedEvent.EventType; foreach (var projection in _projections) { var cname = projection.Info.CommonName; long elapsedticks; try { QueryPerformanceCounter(out long ticks1); await projection.HandleAsync(unwindedEvent.GetEvent(), true).ConfigureAwait(false); QueryPerformanceCounter(out long ticks2); elapsedticks = ticks2 - ticks1; KernelMetricsHelper.IncrementProjectionCounterRebuild(cname, SlotName, eventName, elapsedticks); } catch (Exception ex) { _logger.FatalFormat(ex, "[Slot: {3} Projection: {4}] Failed checkpoint: {0} StreamId: {1} Event Name: {2}", unwindedEvent.CheckpointToken, unwindedEvent.PartitionId, eventName, SlotName, cname ); HealthChecks.RegisterHealthCheck($"RebuildDispatcher, slot {SlotName} - FailedCheckpoint {unwindedEvent.CheckpointToken}", () => HealthCheckResult.Unhealthy(ex) ); throw; } _metrics.Inc(cname, eventName, elapsedticks); if (_logger.IsDebugEnabled) { _logger.DebugFormat("[{3}] [{4}] Handled checkpoint {0}: {1} > {2}", unwindedEvent.CheckpointToken, unwindedEvent.PartitionId, eventName, SlotName, cname ); } } } catch (Exception ex) { _logger.ErrorFormat(ex, "Error dispathing commit id: {0}\nMessage: {1}\nError: {2}", unwindedEvent.CheckpointToken, unwindedEvent.Event, ex.Message); HealthChecks.RegisterHealthCheck($"RebuildDispatcher, slot {SlotName} - GeneralError", () => HealthCheckResult.Unhealthy(ex) ); throw; } _lastCheckpointRebuilded = chkpoint; KernelMetricsHelper.MarkEventInRebuildDispatchedCount(SlotName, 1); Interlocked.Decrement(ref RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit); }
internal void UploadFile(String jobFile, DocumentImportTask task) { String fname = ""; try { TenantContext.Enter(task.Tenant); if (!task.Uri.IsFile) { LogAndThrow("Error importing task file {0}: Uri is not a file: {1}", jobFile, task.Uri); } fname = task.Uri.LocalPath; if (FileHasImportFailureMarker(fname, task.FileTimestamp)) { return; } if (!File.Exists(fname)) { LogAndThrow("Error importing task file {0}: File missing: {1}", jobFile, fname); } var blobStore = GetBlobStoreForTenant(); var identityGenerator = GetIdentityGeneratorForTenant(); if (blobStore == null || identityGenerator == null) { Logger.ErrorFormat("Tenant {1} not found or not configured for file: {1}", task.Tenant, fname); return; } BlobId blobId; if (!String.IsNullOrEmpty(task.FileName)) { //use the real file name from the task not the name of the file using (FileStream fs = File.Open(fname, FileMode.Open, FileAccess.Read)) { blobId = blobStore.Upload(task.Format, new FileNameWithExtension(task.FileName), fs); } } else { //No filename given in task, use name of the blob blobId = blobStore.Upload(task.Format, fname); } if (task.Format == OriginalFormat) { var descriptor = blobStore.GetDescriptor(blobId); var fileName = new FileNameWithExtension(task.FileName); var handleInfo = new DocumentHandleInfo(task.Handle, fileName, task.CustomData); var documentId = identityGenerator.New <DocumentDescriptorId>(); var createDocument = new InitializeDocumentDescriptor( documentId, blobId, handleInfo, descriptor.Hash, fileName ); _commandBus.Send(createDocument, "import-from-file"); } else { var reader = _tenantAccessor.Current.Container.Resolve <IDocumentWriter>(); var handle = reader.FindOneById(task.Handle); var documentId = handle.DocumentDescriptorId; var command = new AddFormatToDocumentDescriptor( documentId, task.Format, blobId, new PipelineId("user-content") ); _commandBus.Send(command, "import-from-file"); } TaskExecuted(task); DeleteImportFailure(fname); } catch (Exception ex) { Logger.ErrorFormat(ex, "Job Import Queue - Error importing {0} - {1}", jobFile, ex.Message); ImportFailure failure = new ImportFailure() { Error = ex.ToString(), FileName = fname, Timestamp = DateTime.Now, ImportFileTimestampTicks = task.FileTimestamp.Ticks, }; MarkImportFailure(failure); } finally { TenantContext.Exit(); } }
/// <summary> /// Start rebuild process. /// </summary> /// <returns></returns> public async Task <RebuildStatus> RebuildAsync() { if (Logger.IsInfoEnabled) { Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId); } DumpProjections(); await _eventUnwinder.UnwindAsync().ConfigureAwait(false); _status = new RebuildStatus(); TenantContext.Enter(_config.TenantId); await ConfigureProjections().ConfigureAwait(false); var allSlots = _projectionsBySlot.Keys.ToArray(); //initialize dispatching of the commits foreach (var bucket in _config.BucketInfo) { _consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>()); _status.AddBucket(); } //Setup the slots foreach (var slotName in allSlots) { Logger.InfoFormat("Slot {0} will be rebuilded", slotName); var projectionsForThisSlot = _projectionsBySlot[slotName]; Int64 maximumDispatchedValue = projectionsForThisSlot .Select(p => _checkpointTracker.GetCheckpoint(p)) .Max(); var dispatcher = new RebuildProjectionSlotDispatcher(Logger, slotName, _config, projectionsForThisSlot, maximumDispatchedValue, _loggerThreadContextManager); KernelMetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch); _rebuildDispatchers.Add(dispatcher); //find right consumer var slotBucket = _config.BucketInfo.SingleOrDefault(b => b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ?? _config.BucketInfo.Single(b => b.Slots[0] == "*"); var consumerList = _consumers[slotBucket]; consumerList.Add(dispatcher); } //Creates TPL chain and start polling everything on the consumer foreach (var consumer in _consumers) { var bucketInfo = String.Join(",", consumer.Key.Slots); if (consumer.Value.Count == 0) { Logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo); _status.BucketDone(bucketInfo, 0, 0, 0); continue; } var consumerBufferOptions = new DataflowBlockOptions(); consumerBufferOptions.BoundedCapacity = consumer.Key.BufferSize; var _buffer = new BufferBlock <UnwindedDomainEvent>(consumerBufferOptions); ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions(); executionOption.BoundedCapacity = consumer.Key.BufferSize; var dispatcherList = consumer.Value; _projectionInspector.ResetHandledEvents(); List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> > consumers = new List <SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> >(); foreach (var dispatcher in dispatcherList) { ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions(); consumerOptions.BoundedCapacity = consumer.Key.BufferSize; var actionBlock = new ActionBlock <UnwindedDomainEvent>((Func <UnwindedDomainEvent, Task>)dispatcher.DispatchEventAsync, consumerOptions); HashSet <Type> eventsOfThisSlot = new HashSet <Type>(); foreach (var projection in dispatcher.Projections) { var domainEventTypesHandledByThisProjection = _projectionInspector.InspectProjectionForEvents(projection.GetType()); foreach (var type in domainEventTypesHandledByThisProjection) { eventsOfThisSlot.Add(type); } } SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent> slotInfo = new SlotGuaranteedDeliveryBroadcastBlock.SlotInfo <UnwindedDomainEvent>( actionBlock, dispatcher.SlotName, eventsOfThisSlot); consumers.Add(slotInfo); KernelMetricsHelper.CreateMeterForRebuildDispatcherBuffer(dispatcher.SlotName, () => actionBlock.InputCount); } var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList(); var broadcaster = SlotGuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, consumer.Key.BufferSize); _buffer.LinkTo(broadcaster, new DataflowLinkOptions() { PropagateCompletion = true }); KernelMetricsHelper.CreateGaugeForRebuildFirstBuffer(bucketInfo, () => _buffer.Count); KernelMetricsHelper.CreateGaugeForRebuildBucketDBroadcasterBuffer(bucketInfo, () => broadcaster.InputCount); KernelMetricsHelper.CreateMeterForRebuildEventCompleted(bucketInfo); //fire each bucket in own thread #pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed Task.Factory.StartNew(() => StartPoll( _buffer, broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers)); //await StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers).ConfigureAwait(false); #pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed } KernelMetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit); return(_status); }
private async Task DispatchCommitAsync(IChunk chunk, string slotName, Int64 startCheckpoint) { Interlocked.Increment(ref _countOfConcurrentDispatchingCommit); _loggerThreadContextManager.SetContextProperty("commit", $"{chunk.OperationId}/{chunk.Position}"); if (_logger.IsDebugEnabled) { _logger.DebugFormat("Dispatching checkpoit {0} on tenant {1}", chunk.Position, _config.TenantId); } TenantContext.Enter(_config.TenantId); var chkpoint = chunk.Position; if (!lastCheckpointDispatched.ContainsKey(slotName)) { lastCheckpointDispatched[slotName] = 0; } if (lastCheckpointDispatched[slotName] >= chkpoint) { var error = String.Format("Sequence broken, last checkpoint for slot {0} was {1} and now we dispatched {2}", slotName, lastCheckpointDispatched[slotName], chkpoint); _logger.Error(error); throw new JarvisFrameworkEngineException(error); } if (lastCheckpointDispatched[slotName] + 1 != chkpoint && lastCheckpointDispatched[slotName] > 0) { _logger.DebugFormat("Sequence of commit not consecutive (probably holes), last dispatched {0} receiving {1}", lastCheckpointDispatched[slotName], chkpoint); } lastCheckpointDispatched[slotName] = chkpoint; if (chkpoint <= startCheckpoint) { //Already dispatched, skip it. Interlocked.Decrement(ref _countOfConcurrentDispatchingCommit); return; } var projections = _projectionsBySlot[slotName]; object[] events = GetArrayOfObjectFromChunk(chunk); var eventCount = events.Length; //now it is time to dispatch the commit, we will dispatch each projection //and for each projection we dispatch all the events present in changeset bool someProjectionProcessedTheEvent = false; foreach (var projection in projections) { var cname = projection.Info.CommonName; Object eventMessage = null; try { //Foreach projection we need to dispach every event for (int index = 0; index < eventCount; index++) { eventMessage = events[index]; var message = eventMessage as IMessage; string eventName = eventMessage.GetType().Name; _loggerThreadContextManager.SetContextProperty("evType", eventName); _loggerThreadContextManager.SetContextProperty("evMsId", message?.MessageId); _loggerThreadContextManager.SetContextProperty("evCheckpointToken", chunk.Position); _loggerThreadContextManager.SetContextProperty("prj", cname); var checkpointStatus = _checkpointTracker.GetCheckpointStatus(cname, chunk.Position); long ticks = 0; try { //pay attention, stopwatch consumes time. var sw = new Stopwatch(); sw.Start(); var eventProcessed = await projection .HandleAsync(eventMessage, checkpointStatus.IsRebuilding) .ConfigureAwait(false); someProjectionProcessedTheEvent |= eventProcessed; sw.Stop(); ticks = sw.ElapsedTicks; KernelMetricsHelper.IncrementProjectionCounter(cname, slotName, eventName, ticks, sw.ElapsedMilliseconds); if (_logger.IsDebugEnabled) { _logger.DebugFormat("[Slot:{3}] [Projection {4}] Handled checkpoint {0}: {1} > {2}", chunk.Position, chunk.PartitionId, $"eventName: {eventName} [event N°{index}]", slotName, cname ); } } catch (Exception ex) { var error = String.Format("[Slot: {3} Projection: {4}] Failed checkpoint: {0} StreamId: {1} Event Name: {2}", chunk.Position, chunk.PartitionId, eventName, slotName, cname); _logger.Fatal(error, ex); _engineFatalErrors.Add(String.Format("{0}\n{1}", error, ex)); throw; } } //End of event cycle } catch (Exception ex) { _loggerThreadContextManager.ClearContextProperty("commit"); _logger.ErrorFormat(ex, "Error dispathing Chunk [{0}]\n Message: {1}\n Error: {2}\n", chunk.Position, eventMessage?.GetType()?.Name, ex.Message); throw; } finally { _loggerThreadContextManager.ClearContextProperty("evType"); _loggerThreadContextManager.ClearContextProperty("evMsId"); _loggerThreadContextManager.ClearContextProperty("evCheckpointToken"); _loggerThreadContextManager.ClearContextProperty("prj"); } projection.CheckpointProjected(chunk.Position); //TODO: Evaluate if it is needed to update single projection checkpoint //Update this projection, set all events of this checkpoint as dispatched. //await _checkpointTracker.UpdateProjectionCheckpointAsync(cname, chunk.Position).ConfigureAwait(false); } //End of projection cycle. await _checkpointTracker.UpdateSlotAndSetCheckpointAsync( slotName, projections.Select(_ => _.Info.CommonName), chunk.Position, someEventDispatched : someProjectionProcessedTheEvent).ConfigureAwait(false); KernelMetricsHelper.MarkCommitDispatchedCount(slotName, 1); await _notifyCommitHandled.SetDispatched(slotName, chunk).ConfigureAwait(false); // ok in multithread wihout locks! if (_maxDispatchedCheckpoint < chkpoint) { if (_logger.IsDebugEnabled) { _logger.DebugFormat("Updating last dispatched checkpoint from {0} to {1}", _maxDispatchedCheckpoint, chkpoint ); } _maxDispatchedCheckpoint = chkpoint; } _loggerThreadContextManager.ClearContextProperty("commit"); Interlocked.Decrement(ref _countOfConcurrentDispatchingCommit); }
void DispatchCommit(ICommit commit, string slotName, Int64 startCheckpoint) { Interlocked.Increment(ref _countOfConcurrentDispatchingCommit); if (Logger.IsDebugEnabled) { Logger.ThreadProperties["commit"] = commit.CommitId; } if (Logger.IsDebugEnabled) { Logger.DebugFormat("Dispatching checkpoit {0} on tenant {1}", commit.CheckpointToken, _config.TenantId); } TenantContext.Enter(_config.TenantId); var chkpoint = commit.CheckpointToken; if (!lastCheckpointDispatched.ContainsKey(slotName)) { lastCheckpointDispatched[slotName] = 0; } if (lastCheckpointDispatched[slotName] >= chkpoint) { var error = String.Format("Sequence broken, last checkpoint for slot {0} was {1} and now we dispatched {2}", slotName, lastCheckpointDispatched[slotName], chkpoint); Logger.Error(error); throw new ApplicationException(error); } if (lastCheckpointDispatched[slotName] + 1 != chkpoint) { if (lastCheckpointDispatched[slotName] > 0) { Logger.DebugFormat("Sequence of commit non consecutive, last dispatched {0} receiving {1}", lastCheckpointDispatched[slotName], chkpoint); } } lastCheckpointDispatched[slotName] = chkpoint; if (chkpoint <= startCheckpoint) { //Already dispatched, skip it. Interlocked.Decrement(ref _countOfConcurrentDispatchingCommit); return; } var projections = _projectionsBySlot[slotName]; bool dispatchCommit = false; var eventCount = commit.Events.Count; var projectionToUpdate = new List <string>(); for (int index = 0; index < eventCount; index++) { var eventMessage = commit.Events.ElementAt(index); try { var evt = (DomainEvent)eventMessage.Body; if (Logger.IsDebugEnabled) { Logger.ThreadProperties["evType"] = evt.GetType().Name; } if (Logger.IsDebugEnabled) { Logger.ThreadProperties["evMsId"] = evt.MessageId; } if (Logger.IsDebugEnabled) { Logger.ThreadProperties["evCheckpointToken"] = commit.CheckpointToken; } string eventName = evt.GetType().Name; foreach (var projection in projections) { var cname = projection.GetCommonName(); if (Logger.IsDebugEnabled) { Logger.ThreadProperties["prj"] = cname; } var checkpointStatus = _checkpointTracker.GetCheckpointStatus(cname, commit.CheckpointToken); bool handled; long ticks = 0; try { //pay attention, stopwatch consumes time. var sw = new Stopwatch(); sw.Start(); handled = projection.Handle(evt, checkpointStatus.IsRebuilding); sw.Stop(); ticks = sw.ElapsedTicks; MetricsHelper.IncrementProjectionCounter(cname, slotName, eventName, ticks); } catch (Exception ex) { var error = String.Format("[Slot: {3} Projection: {4}] Failed checkpoint: {0} StreamId: {1} Event Name: {2}", commit.CheckpointToken, commit.StreamId, eventName, slotName, cname); Logger.Fatal(error, ex); _engineFatalErrors.Add(error); throw; } if (handled) { _metrics.Inc(cname, eventName, ticks); if (Logger.IsDebugEnabled) { Logger.DebugFormat("[{3}] [{4}] Handled checkpoint {0}: {1} > {2}", commit.CheckpointToken, commit.StreamId, eventName, slotName, cname ); } } if (!checkpointStatus.IsRebuilding) { if (handled) { dispatchCommit = true; } projectionToUpdate.Add(cname); } else { if (checkpointStatus.IsLast && (index == eventCount - 1)) { projection.StopRebuild(); var meter = _metrics.GetMeter(cname); _checkpointTracker.RebuildEnded(projection, meter); Logger.InfoFormat("Rebuild done {0} @ {1}", projection.GetType().FullName, commit.CheckpointToken ); } } if (Logger.IsDebugEnabled) { Logger.ThreadProperties["prj"] = null; } } ClearLoggerThreadPropertiesForEventDispatchLoop(); } catch (Exception ex) { Logger.ErrorFormat(ex, "Error dispathing commit id: {0}\nMessage: {1}\nError: {2}", commit.CheckpointToken, eventMessage.Body, ex.Message); ClearLoggerThreadPropertiesForEventDispatchLoop(); throw; } } if (projectionToUpdate.Count == 0) { //I'm in rebuilding or no projection had run any events, only update slot _checkpointTracker.UpdateSlot(slotName, commit.CheckpointToken); } else { //I'm not on rebuilding, we have projection to update, update everything with one call. _checkpointTracker.UpdateSlotAndSetCheckpoint(slotName, projectionToUpdate, commit.CheckpointToken); } MetricsHelper.MarkCommitDispatchedCount(slotName, 1); if (dispatchCommit) { _notifyCommitHandled.SetDispatched(commit); } // ok in multithread wihout locks! if (_maxDispatchedCheckpoint < chkpoint) { if (Logger.IsDebugEnabled) { Logger.DebugFormat("Updating last dispatched checkpoint from {0} to {1}", _maxDispatchedCheckpoint, chkpoint ); } _maxDispatchedCheckpoint = chkpoint; } if (Logger.IsDebugEnabled) { Logger.ThreadProperties["commit"] = null; } Interlocked.Decrement(ref _countOfConcurrentDispatchingCommit); }
void DispatchCommit(ICommit commit, string slotName, LongCheckpoint startCheckpoint) { if (Logger.IsDebugEnabled) { Logger.ThreadProperties["commit"] = commit.CommitId; } if (Logger.IsDebugEnabled) { Logger.DebugFormat("Dispatching checkpoit {0} on tenant {1}", commit.CheckpointToken, _config.TenantId); } TenantContext.Enter(_config.TenantId); var chkpoint = LongCheckpoint.Parse(commit.CheckpointToken); if (chkpoint.LongValue <= startCheckpoint.LongValue) { // già fatta return; } var projections = _projectionsBySlot[slotName]; bool dispatchCommit = false; var eventMessages = commit.Events.ToArray(); var projectionToUpdate = new List <string>(); for (int index = 0; index < eventMessages.Length; index++) { var eventMessage = eventMessages[index]; try { var evt = (DomainEvent)eventMessage.Body; if (Logger.IsDebugEnabled) { Logger.ThreadProperties["evType"] = evt.GetType().Name; } if (Logger.IsDebugEnabled) { Logger.ThreadProperties["evMsId"] = evt.MessageId; } string eventName = evt.GetType().Name; foreach (var projection in projections) { var cname = projection.GetCommonName(); if (Logger.IsDebugEnabled) { Logger.ThreadProperties["prj"] = cname; } var checkpointStatus = _checkpointTracker.GetCheckpointStatus(cname, commit.CheckpointToken); bool handled; long ticks = 0; try { var sw = new Stopwatch(); sw.Start(); //if (Logger.IsDebugEnabled)Logger.DebugFormat("Handling {0} with {1}", commit.CheckpointToken, projection.GetType().Name); handled = projection.Handle(evt, checkpointStatus.IsRebuilding); //if (Logger.IsDebugEnabled)Logger.DebugFormat("Handled {0} with {1}", commit.CheckpointToken, projection.GetType().Name); sw.Stop(); ticks = sw.ElapsedTicks; } catch (Exception ex) { Logger.FatalFormat(ex, "[{3}] Failed checkpoint {0}: {1} > {2}", commit.CheckpointToken, commit.StreamId, eventName, slotName ); throw; } if (handled) { _metrics.Inc(cname, eventName, ticks); Logger.DebugFormat("[{3}] [{4}] Handled checkpoint {0}: {1} > {2}", commit.CheckpointToken, commit.StreamId, eventName, slotName, cname ); } if (!checkpointStatus.IsRebuilding) { if (handled) { dispatchCommit = true; } projectionToUpdate.Add(cname); } else { if (checkpointStatus.IsLast && (index == eventMessages.Length - 1)) { projection.StopRebuild(); var meter = _metrics.GetMeter(cname); _checkpointTracker.RebuildEnded(projection, meter); Logger.InfoFormat("Rebuild done {0} @ {1}", projection.GetType().FullName, commit.CheckpointToken ); } } if (Logger.IsDebugEnabled) { Logger.ThreadProperties["prj"] = null; } } if (Logger.IsDebugEnabled) { Logger.ThreadProperties["evType"] = null; } if (Logger.IsDebugEnabled) { Logger.ThreadProperties["evMsId"] = null; } } catch (Exception ex) { if (Logger.IsDebugEnabled) { Logger.ErrorFormat(ex, "{0} -> {1}", eventMessage.Body, ex.Message); } throw; } } _checkpointTracker.UpdateSlot(slotName, commit.CheckpointToken); MetricsHelper.MarkCommitDispatchedCount(slotName, 1); foreach (var cname in projectionToUpdate) { _checkpointTracker.SetCheckpoint(cname, commit.CheckpointToken); } if (dispatchCommit) { _notifyCommitHandled.SetDispatched(commit); } // ok in multithread wihout locks! if (_maxDispatchedCheckpoint < chkpoint.LongValue) { if (Logger.IsDebugEnabled) { Logger.DebugFormat("Updating last dispatched checkpoint from {0} to {1}", _maxDispatchedCheckpoint, chkpoint.LongValue ); } _maxDispatchedCheckpoint = chkpoint.LongValue; } if (Logger.IsDebugEnabled) { Logger.ThreadProperties["commit"] = null; } }
public RebuildStatus Rebuild() { if (Logger.IsInfoEnabled) { Logger.InfoFormat("Starting rebuild projection engine on tenant {0}", _config.TenantId); } DumpProjections(); _eventUnwinder.Unwind(); _status = new RebuildStatus(); TenantContext.Enter(_config.TenantId); ConfigureProjections(); var allSlots = _projectionsBySlot.Keys.ToArray(); //initialize dispatching of the commits foreach (var bucket in _config.BucketInfo) { _consumers.Add(bucket, new List <RebuildProjectionSlotDispatcher>()); _status.AddBucket(); } //Setup the slots foreach (var slotName in allSlots) { var startCheckpoint = GetStartCheckpointForSlot(slotName); Logger.InfoFormat("Slot {0} starts from {1}", slotName, startCheckpoint); var projectionsForThisSlot = _projectionsBySlot[slotName]; Int64 maximumDispatchedValue = projectionsForThisSlot .Select(p => _checkpointTracker.GetCheckpoint(p)) .Max(); var dispatcher = new RebuildProjectionSlotDispatcher(_logger, slotName, _config, projectionsForThisSlot, _checkpointTracker, maximumDispatchedValue); MetricsHelper.SetCheckpointCountToDispatch(slotName, () => dispatcher.CheckpointToDispatch); _rebuildDispatchers.Add(dispatcher); //find right consumer var slotBucket = _config.BucketInfo.SingleOrDefault(b => b.Slots.Any(s => s.Equals(slotName, StringComparison.OrdinalIgnoreCase))) ?? _config.BucketInfo.Single(b => b.Slots[0] == "*"); var consumerList = _consumers[slotBucket]; consumerList.Add(dispatcher); } //now start tpl and start polling in other threads. foreach (var consumer in _consumers) { var bucketInfo = String.Join(",", consumer.Key.Slots); if (consumer.Value.Count == 0) { _logger.InfoFormat("Bucket {0} has no active slot, and will be ignored!", bucketInfo); _status.BucketDone(bucketInfo, 0, 0, 0); continue; } DataflowBlockOptions consumerBufferOptions = new DataflowBlockOptions(); consumerBufferOptions.BoundedCapacity = _bufferSize; var _buffer = new BufferBlock <DomainEvent>(consumerBufferOptions); ExecutionDataflowBlockOptions executionOption = new ExecutionDataflowBlockOptions(); executionOption.BoundedCapacity = _bufferSize; var dispatcherList = consumer.Value; _projectionInspector.ResetHandledEvents(); List <ActionBlock <DomainEvent> > consumers = new List <ActionBlock <DomainEvent> >(); foreach (var dispatcher in dispatcherList) { ExecutionDataflowBlockOptions consumerOptions = new ExecutionDataflowBlockOptions(); consumerOptions.BoundedCapacity = _bufferSize; var actionBlock = new ActionBlock <DomainEvent>((Action <DomainEvent>)dispatcher.DispatchEvent, consumerOptions); consumers.Add(actionBlock); foreach (var projection in dispatcher.Projections) { _projectionInspector.InspectProjectionForEvents(projection.GetType()); } } var allTypeHandledStringList = _projectionInspector.EventHandled.Select(t => t.Name).ToList(); var _broadcaster = GuaranteedDeliveryBroadcastBlock.Create(consumers, bucketInfo, 3000); _buffer.LinkTo(_broadcaster, new DataflowLinkOptions() { PropagateCompletion = true }); Task.Factory.StartNew(() => StartPoll(_buffer, _broadcaster, bucketInfo, dispatcherList, allTypeHandledStringList, consumers)); } MetricsHelper.SetProjectionEngineCurrentDispatchCount(() => RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit); return(_status); }