/// <summary> /// The action that is taken for each listener report /// </summary> /// <param name="report"></param> protected virtual void EvaluateListenerReport(Message <ListenerReport> report) { try { if (report.Data.Count > 1) { // this method assumes that only one report is contained per message, so die here - something's gotten messed up throw new InvalidOperationException( "More than the expected message count was received. This indicates a bug in the program."); } var reportData = report.Data.First(); var queueConfiguration = _queueConfigurations.FirstOrDefault(x => x.QueueIdentifier == reportData.Queue); if (queueConfiguration == null && _dynamicallyKeyedWorkerConfigurations.ContainsKey(reportData.Queue)) { // A request-driven queue, which means we need to... var dynamicWorker = _dynamicallyKeyedWorkerConfigurations[reportData.Queue]; _actorManager.CreateAndStartKeyedSoloWorker(dynamicWorker, _dependencyResolver, dynamicWorker.Key, dynamicWorker.KeyType); return; } var numActiveWorkers = _actorManager.GetWorkerCount(reportData.Queue); var numWorkersRequired = Utilities.GetNumWorkersRequiredToClearQueue(queueConfiguration, _statisticsProvider, reportData.MessageCount); if (numActiveWorkers >= numWorkersRequired) { // We gucci return; } var externalSystemStates = new List <DependencyInfo>(); Parallel.ForEach(_externalDependencies, dependency => { externalSystemStates.Add(dependency.GetCurrentState()); }); var canScale = externalSystemStates.All(x => x.CanScaleUp); var mostLimitedSystem = externalSystemStates.OrderBy(x => x.PercentAvailableResources).FirstOrDefault(); if (!canScale && numActiveWorkers > 0) { // One of our dependencies is resource-capped, so we're going to wait until things settle down return; } if (!canScale) { // If a dependency is reporting that it can't take any additional load, but we don't have *any* workers, make one // We don't want to STOP processing, just slow down numWorkersRequired = 1; } var numNewWorkers = numWorkersRequired - numActiveWorkers; var affectedWorkflow = _orderedWorkflowQueueIdentifiers.FirstOrDefault(x => x.Contains(reportData.Queue)) ?? new List <string>(); numNewWorkers = Utilities.GetNumWorkersToScaleWith(numNewWorkers, reportData.Queue, mostLimitedSystem, affectedWorkflow); var workerConfiguration = GetWorkerConfiguration(reportData.Queue); for (var i = 0; i < numNewWorkers; i++) { if (numActiveWorkers >= _actorManager.MaxNumWorkersPerGroup) { break; } _actorManager.CreateAndStartWorker(workerConfiguration, _dependencyResolver); numActiveWorkers++; } } catch (Exception ex) { OnErrorAction(ex); } }