internal static ScaleVote GetAggregateScaleVote(List <ScaleVote> votes, ScaleStatusContext context, ILogger logger)
{
ScaleVote vote = ScaleVote.None;
if (votes.Any())
{
// aggregate all the votes into a single vote
if (votes.Any(p => p == ScaleVote.ScaleOut))
{
// scale out if at least 1 monitor requires it
logger.LogDebug("Scaling out based on votes");
vote = ScaleVote.ScaleOut;
}
else if (context.WorkerCount > 0 && votes.All(p => p == ScaleVote.ScaleIn))
{
// scale in only if all monitors vote scale in
logger.LogDebug("Scaling in based on votes");
vote = ScaleVote.ScaleIn;
}
}
else if (context.WorkerCount > 0)
{
// if no functions exist or are enabled we'll scale in
logger.LogDebug("No enabled functions or scale votes so scaling in");
vote = ScaleVote.ScaleIn;
}
return(vote);
}
public void GetScaleStatus_QueueSteady_ReturnsVote_None()
{
var context = new ScaleStatusContext <RabbitMQTriggerMetrics>
{
WorkerCount = 2
};
var Timestamp = DateTime.UtcNow;
context.Metrics = new List <RabbitMQTriggerMetrics>
{
new RabbitMQTriggerMetrics {
QueueLength = 1500, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 1600, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 1400, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 1300, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 1700, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 1600, Timestamp = Timestamp.AddSeconds(15)
}
};
var status = _testListener.GetScaleStatus(context);
Assert.Equal(ScaleVote.None, status.Vote);
}
public void GetScaleStatus_QueueIdle_ReturnsVote_ScaleOut()
{
var context = new ScaleStatusContext <RabbitMQTriggerMetrics>
{
WorkerCount = 3
};
var Timestamp = DateTime.UtcNow;
context.Metrics = new List <RabbitMQTriggerMetrics>
{
new RabbitMQTriggerMetrics {
QueueLength = 0, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 0, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 0, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 0, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 0, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 0, Timestamp = Timestamp.AddSeconds(15)
}
};
var status = _testListener.GetScaleStatus(context);
Assert.Equal(ScaleVote.ScaleIn, status.Vote);
}
public async Task GetScaleStatus_MonitorFails_ReturnsExpectedResult()
{
var context = new ScaleStatusContext
{
WorkerCount = 3
};
var mockMonitor1 = new Mock <IScaleMonitor>(MockBehavior.Strict);
mockMonitor1.Setup(p => p.GetScaleStatus(It.Is <ScaleStatusContext>(q => q.WorkerCount == context.WorkerCount))).Returns(new ScaleStatus {
Vote = ScaleVote.ScaleIn
});
mockMonitor1.SetupGet(p => p.Descriptor).Returns(new ScaleMonitorDescriptor("testscalemonitor1"));
var mockMonitor2 = new Mock <IScaleMonitor>(MockBehavior.Strict);
mockMonitor2.SetupGet(p => p.Descriptor).Returns(new ScaleMonitorDescriptor("testscalemonitor2"));
var exception = new Exception("Kaboom!");
mockMonitor2.Setup(p => p.GetScaleStatus(It.Is <ScaleStatusContext>(q => q.WorkerCount == context.WorkerCount))).Throws(exception);
var mockMonitor3 = new Mock <IScaleMonitor>(MockBehavior.Strict);
mockMonitor3.Setup(p => p.GetScaleStatus(It.Is <ScaleStatusContext>(q => q.WorkerCount == context.WorkerCount))).Returns(new ScaleStatus {
Vote = ScaleVote.ScaleIn
});
mockMonitor3.SetupGet(p => p.Descriptor).Returns(new ScaleMonitorDescriptor("testscalemonitor3"));
List <IScaleMonitor> monitors = new List <IScaleMonitor>
{
mockMonitor1.Object,
mockMonitor2.Object,
mockMonitor3.Object
};
_monitorManagerMock.Setup(p => p.GetMonitors()).Returns(monitors);
var monitorMetrics = new Dictionary <IScaleMonitor, IList <ScaleMetrics> >
{
{ mockMonitor1.Object, new List <ScaleMetrics>() },
{ mockMonitor2.Object, new List <ScaleMetrics>() },
{ mockMonitor3.Object, new List <ScaleMetrics>() }
};
_metricsRepositoryMock.Setup(p => p.ReadMetricsAsync(It.IsAny <IEnumerable <IScaleMonitor> >())).ReturnsAsync(monitorMetrics);
var status = await _scaleManager.GetScaleStatusAsync(context);
var logs = _loggerProvider.GetAllLogMessages();
Assert.Equal("Computing scale status (WorkerCount=3)", logs[0].FormattedMessage);
Assert.Equal("3 scale monitors to sample", logs[1].FormattedMessage);
Assert.Equal("Monitor 'testscalemonitor1' voted 'ScaleIn'", logs[2].FormattedMessage);
Assert.Equal("Failed to query scale status for monitor 'testscalemonitor2'.", logs[3].FormattedMessage);
Assert.Same(exception, logs[3].Exception);
Assert.Equal("Monitor 'testscalemonitor3' voted 'ScaleIn'", logs[4].FormattedMessage);
Assert.Equal(ScaleVote.ScaleIn, status.Vote);
}
public async Task GetScaleStatus_NoMonitors_ReturnsExpectedStatus(int workerCount, ScaleVote expected)
{
var context = new ScaleStatusContext
{
WorkerCount = workerCount
};
var status = await _scaleManager.GetScaleStatusAsync(context);
Assert.Equal(expected, status.Vote);
}
/// <summary>
/// Get the current scale status (vote) by querying all active monitors for their
/// scale status.
/// </summary>
/// <param name="context">The context to use for the scale decision.</param>
/// <returns>The scale vote.</returns>
public virtual async Task <ScaleStatusResult> GetScaleStatusAsync(ScaleStatusContext context)
{
var monitors = _monitorManager.GetMonitors();
List <ScaleVote> votes = new List <ScaleVote>();
if (monitors.Any())
{
// get the collection of current metrics for each monitor
var monitorMetrics = await _metricsRepository.ReadMetricsAsync(monitors);
_logger.LogDebug($"Computing scale status (WorkerCount={context.WorkerCount})");
_logger.LogDebug($"{monitorMetrics.Count} scale monitors to sample");
// for each monitor, ask it to return its scale status (vote) based on
// the metrics and context info (e.g. worker count)
foreach (var pair in monitorMetrics)
{
var monitor = pair.Key;
var metrics = pair.Value;
try
{
// create a new context instance to avoid modifying the
// incoming context
var scaleStatusContext = new ScaleStatusContext
{
WorkerCount = context.WorkerCount,
Metrics = metrics
};
var result = monitor.GetScaleStatus(scaleStatusContext);
_logger.LogDebug($"Monitor '{monitor.Descriptor.Id}' voted '{result.Vote.ToString()}'");
votes.Add(result.Vote);
}
catch (Exception exc) when(!exc.IsFatal())
{
// if a particular monitor fails, log and continue
_logger.LogError(exc, $"Failed to query scale status for monitor '{monitor.Descriptor.Id}'.");
}
}
}
else
{
// no monitors registered
// this can happen if the host is offline
}
var vote = GetAggregateScaleVote(votes, context, _logger);
return(new ScaleStatusResult
{
Vote = vote
});
}
public void GetScaleStatus_MessagesPerWorkerThresholdExceeded_ReturnsVote_ScaleOut()
{
var context = new ScaleStatusContext <ServiceBusTriggerMetrics>
{
WorkerCount = 1
};
var timestamp = DateTime.UtcNow;
var serviceBusTriggerMetrics = new List <ServiceBusTriggerMetrics>
{
new ServiceBusTriggerMetrics {
MessageCount = 2500, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(0), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 2505, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(0), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 2612, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(0), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 2700, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(0), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 2810, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(0), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 2900, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(0), Timestamp = timestamp.AddSeconds(15)
},
};
context.Metrics = serviceBusTriggerMetrics;
var status = _scaleMonitor.GetScaleStatus(context);
Assert.AreEqual(ScaleVote.ScaleOut, status.Vote);
var logs = _loggerProvider.GetAllLogMessages().ToArray();
var log = logs[0];
Assert.AreEqual(LogLevel.Information, log.Level);
Assert.AreEqual("MessageCount (2900) > WorkerCount (1) * 1,000.", log.FormattedMessage);
log = logs[1];
Assert.AreEqual(LogLevel.Information, log.Level);
Assert.AreEqual($"Message count for Service Bus Entity ({_entityPath}, 2900) " +
$"is too high relative to the number of instances (1).", log.FormattedMessage);
// verify again with a non generic context instance
var context2 = new ScaleStatusContext
{
WorkerCount = 1,
Metrics = serviceBusTriggerMetrics
};
status = ((IScaleMonitor)_scaleMonitor).GetScaleStatus(context2);
Assert.AreEqual(ScaleVote.ScaleOut, status.Vote);
}
public void GetScaleStatus_EventsPerWorkerThresholdExceeded_ReturnsVote_ScaleOut()
{
var context = new ScaleStatusContext <EventHubsTriggerMetrics>
{
WorkerCount = 1
};
var timestamp = DateTime.UtcNow;
var eventHubTriggerMetrics = new List <EventHubsTriggerMetrics>
{
new EventHubsTriggerMetrics {
EventCount = 2500, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 2505, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 2612, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 2700, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 2810, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 2900, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
};
context.Metrics = eventHubTriggerMetrics;
var status = _scaleMonitor.GetScaleStatus(context);
Assert.AreEqual(ScaleVote.ScaleOut, status.Vote);
var logs = _loggerProvider.GetAllLogMessages().ToArray();
var log = logs[0];
Assert.AreEqual(LogLevel.Information, log.Level);
Assert.AreEqual("EventCount (2900) > WorkerCount (1) * 1,000.", log.FormattedMessage);
log = logs[1];
Assert.AreEqual(LogLevel.Information, log.Level);
Assert.AreEqual($"Event count (2900) for EventHubs entity ({_eventHubName}, {_consumerGroup}) " +
$"is too high relative to the number of instances (1).", log.FormattedMessage);
// verify again with a non generic context instance
var context2 = new ScaleStatusContext
{
WorkerCount = 1,
Metrics = eventHubTriggerMetrics
};
status = ((IScaleMonitor)_scaleMonitor).GetScaleStatus(context2);
Assert.AreEqual(ScaleVote.ScaleOut, status.Vote);
}
public void GetScaleStatus_HandlesMalformedMetrics()
{
// Null metrics
var context = new ScaleStatusContext <DurableTaskTriggerMetrics>
{
WorkerCount = 1,
Metrics = null,
};
var recommendation = this.scaleMonitor.GetScaleStatus(context);
Assert.Equal(ScaleVote.None, recommendation.Vote);
// Empty metrics
var heartbeats = new PerformanceHeartbeat[0];
context.Metrics = new DurableTaskTriggerMetrics[0];
this.performanceMonitor
.Setup(m => m.MakeScaleRecommendation(1, heartbeats))
.Returns <ScaleRecommendation>(null);
recommendation = this.scaleMonitor.GetScaleStatus(context);
Assert.Equal(ScaleVote.None, recommendation.Vote);
// Metrics with null properties
var metrics = new DurableTaskTriggerMetrics[5];
for (int i = 0; i < metrics.Length; ++i)
{
metrics[i] = new DurableTaskTriggerMetrics();
}
context.Metrics = metrics;
heartbeats = new PerformanceHeartbeat[5];
for (int i = 0; i < heartbeats.Length; ++i)
{
heartbeats[i] = new PerformanceHeartbeat
{
ControlQueueLengths = new List <int>(),
ControlQueueLatencies = new List <TimeSpan>(),
};
}
this.performanceMonitor
.Setup(m => m.MakeScaleRecommendation(1, this.MatchEquivalentHeartbeats(heartbeats)))
.Returns <ScaleRecommendation>(null);
recommendation = this.scaleMonitor.GetScaleStatus(context);
Assert.Equal(ScaleVote.None, recommendation.Vote);
}
public async Task GetScaleStatus_RuntimeScaleModeNotEnabled_ReturnsBadRequest()
{
var context = new ScaleStatusContext
{
WorkerCount = 5
};
var scaleManagerMock = new Mock <FunctionsScaleManager>(MockBehavior.Strict);
var result = (BadRequestObjectResult)(await _hostController.GetScaleStatus(context, scaleManagerMock.Object));
Assert.Equal(HttpStatusCode.BadRequest, (HttpStatusCode)result.StatusCode);
Assert.Equal("Runtime scale monitoring is not enabled.", result.Value);
}
public async Task <IActionResult> GetScaleStatus([FromBody] ScaleStatusContext context, [FromServices] FunctionsScaleManager scaleManager)
{
// if runtime scale isn't enabled return error
if (!_environment.IsRuntimeScaleMonitoringEnabled())
{
return(BadRequest("Runtime scale monitoring is not enabled."));
}
var scaleStatus = await scaleManager.GetScaleStatusAsync(context);
return(new ObjectResult(scaleStatus));
}
public void GetScaleStatus_NoMetrics_ReturnsVote_None()
{
var context = new ScaleStatusContext <RabbitMQTriggerMetrics>
{
WorkerCount = 1
};
var status = _testListener.GetScaleStatus(context);
Assert.Equal(ScaleVote.None, status.Vote);
status = ((IScaleMonitor)_testListener).GetScaleStatus(context);
Assert.Equal(ScaleVote.None, status.Vote);
}
public void GetScaleStatus_NoMetrics_ReturnsVote_None()
{
var context = new ScaleStatusContext <EventHubsTriggerMetrics>
{
WorkerCount = 1
};
var status = _scaleMonitor.GetScaleStatus(context);
Assert.AreEqual(ScaleVote.None, status.Vote);
// verify the non-generic implementation works properly
status = ((IScaleMonitor)_scaleMonitor).GetScaleStatus(context);
Assert.AreEqual(ScaleVote.None, status.Vote);
}
public async Task GetScaleStatus_FunctionsScaleManager_Null_ReturnsServiceUnavailable()
{
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionsRuntimeScaleMonitoringEnabled)).Returns("1");
var context = new ScaleStatusContext
{
WorkerCount = 5
};
var scriptHostManagerMock = new Mock <IScriptHostManager>(MockBehavior.Strict);
var serviceProviderMock = scriptHostManagerMock.As <IServiceProvider>();
serviceProviderMock.Setup(p => p.GetService(typeof(FunctionsScaleManager))).Returns(null);
var result = (StatusCodeResult)(await _hostController.GetScaleStatus(context, scriptHostManagerMock.Object));
Assert.Equal(StatusCodes.Status503ServiceUnavailable, result.StatusCode);
}
public void GetScaleStatus_MessagesPerWorkerThresholdExceeded_ReturnsVote_ScaleOut()
{
var context = new ScaleStatusContext <RabbitMQTriggerMetrics>
{
WorkerCount = 1
};
var Timestamp = DateTime.UtcNow;
var RabbitMQTriggerMetrics = new List <RabbitMQTriggerMetrics>
{
new RabbitMQTriggerMetrics {
QueueLength = 2500, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 2505, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 2612, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 2700, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 2810, Timestamp = Timestamp.AddSeconds(15)
},
new RabbitMQTriggerMetrics {
QueueLength = 2900, Timestamp = Timestamp.AddSeconds(15)
}
};
context.Metrics = RabbitMQTriggerMetrics;
var status = _testListener.GetScaleStatus(context);
Assert.Equal(ScaleVote.ScaleOut, status.Vote);
// verify again with a non generic context instance
var context2 = new ScaleStatusContext
{
WorkerCount = 1,
Metrics = RabbitMQTriggerMetrics
};
status = ((IScaleMonitor)_testListener).GetScaleStatus(context2);
Assert.Equal(ScaleVote.ScaleOut, status.Vote);
}
public async Task GetScaleStatus_RuntimeScaleModeEnabled_Succeeds()
{
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionsRuntimeScaleMonitoringEnabled)).Returns("1");
var context = new ScaleStatusContext
{
WorkerCount = 5
};
var scaleManagerMock = new Mock <FunctionsScaleManager>(MockBehavior.Strict);
var scaleStatusResult = new ScaleStatusResult {
Vote = ScaleVote.ScaleOut
};
scaleManagerMock.Setup(p => p.GetScaleStatusAsync(context)).ReturnsAsync(scaleStatusResult);
var result = (ObjectResult)(await _hostController.GetScaleStatus(context, scaleManagerMock.Object));
Assert.Same(result.Value, scaleStatusResult);
}
public void GetScaleStatus_MessagesPerWorkerThresholdExceeded_ReturnsVote_ScaleOut()
{
var context = new ScaleStatusContext <CosmosDBTriggerMetrics>
{
WorkerCount = 1
};
var timestamp = DateTime.UtcNow;
context.Metrics = new List <CosmosDBTriggerMetrics>
{
new CosmosDBTriggerMetrics {
RemainingWork = 2500, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new CosmosDBTriggerMetrics {
RemainingWork = 2505, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new CosmosDBTriggerMetrics {
RemainingWork = 2612, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new CosmosDBTriggerMetrics {
RemainingWork = 2700, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new CosmosDBTriggerMetrics {
RemainingWork = 2810, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new CosmosDBTriggerMetrics {
RemainingWork = 2900, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
};
var status = _listener.GetScaleStatus(context);
Assert.Equal(ScaleVote.ScaleOut, status.Vote);
var logs = _loggerProvider.GetAllLogMessages().ToArray();
var log = logs[0];
Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Information, log.Level);
Assert.Equal("RemainingWork (2900) > WorkerCount (1) * 1,000.", log.FormattedMessage);
log = logs[1];
Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Information, log.Level);
Assert.Equal("Remaining work for collection (MonitoredCollection, 2900) is too high relative to the number of instances (1).", log.FormattedMessage);
}
public void GetScaleStatus_ConsistentRemainingWork_ReturnsVote_ScaleOut()
{
var context = new ScaleStatusContext <CosmosDBTriggerMetrics>
{
WorkerCount = 1
};
var timestamp = DateTime.UtcNow;
context.Metrics = new List <CosmosDBTriggerMetrics>
{
new CosmosDBTriggerMetrics {
RemainingWork = 10, PartitionCount = 2, Timestamp = timestamp.AddSeconds(15)
},
new CosmosDBTriggerMetrics {
RemainingWork = 10, PartitionCount = 2, Timestamp = timestamp.AddSeconds(15)
},
new CosmosDBTriggerMetrics {
RemainingWork = 10, PartitionCount = 2, Timestamp = timestamp.AddSeconds(15)
},
new CosmosDBTriggerMetrics {
RemainingWork = 10, PartitionCount = 2, Timestamp = timestamp.AddSeconds(15)
},
new CosmosDBTriggerMetrics {
RemainingWork = 10, PartitionCount = 2, Timestamp = timestamp.AddSeconds(15)
},
new CosmosDBTriggerMetrics {
RemainingWork = 10, PartitionCount = 2, Timestamp = timestamp.AddSeconds(15)
},
};
var status = _listener.GetScaleStatus(context);
Assert.Equal(ScaleVote.ScaleOut, status.Vote);
var logs = _loggerProvider.GetAllLogMessages().ToArray();
var log = logs[0];
Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Information, log.Level);
Assert.Equal("CosmosDB collection 'MonitoredCollection' has documents waiting to be processed.", log.FormattedMessage);
log = logs[1];
Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Information, log.Level);
Assert.Equal("There are 1 instances relative to 2 partitions.", log.FormattedMessage);
}
public void GetAggregateScaleVote_ReturnsExpectedResult(int workerCount, int numScaleOutVotes, int numScaleInVotes, ScaleVote expected)
{
var context = new ScaleStatusContext
{
WorkerCount = workerCount
};
List <ScaleVote> votes = new List <ScaleVote>();
for (int i = 0; i < numScaleOutVotes; i++)
{
votes.Add(ScaleVote.ScaleOut);
}
for (int i = 0; i < numScaleInVotes; i++)
{
votes.Add(ScaleVote.ScaleIn);
}
var vote = FunctionsScaleManager.GetAggregateScaleVote(votes, context, _testLogger);
Assert.Equal(expected, vote);
}
public void When_LagIncreasing_Should_Vote_Scale_Up()
{
var context = new ScaleStatusContext <KafkaTriggerMetrics>()
{
Metrics = new KafkaTriggerMetrics[]
{
new KafkaTriggerMetrics(10, partitions.Count),
new KafkaTriggerMetrics(212, partitions.Count),
new KafkaTriggerMetrics(32323, partitions.Count),
new KafkaTriggerMetrics(121222, partitions.Count),
new KafkaTriggerMetrics(123456, partitions.Count),
},
WorkerCount = 1,
};
var result = topicScaler.GetScaleStatus(context);
Assert.NotNull(result);
Assert.Equal(ScaleVote.ScaleOut, result.Vote);
}
public void When_WorkerCount_More_PartitionCount_Should_Vote_Scale_Down()
{
var context = new ScaleStatusContext <KafkaTriggerMetrics>()
{
Metrics = new KafkaTriggerMetrics[]
{
new KafkaTriggerMetrics(5, partitions.Count),
new KafkaTriggerMetrics(6, partitions.Count),
new KafkaTriggerMetrics(76, partitions.Count),
new KafkaTriggerMetrics(22, partitions.Count),
new KafkaTriggerMetrics(11, partitions.Count),
},
WorkerCount = partitions.Count + 1,
};
var result = topicScaler.GetScaleStatus(context);
Assert.NotNull(result);
Assert.Equal(ScaleVote.ScaleIn, result.Vote);
}
public void When_Not_Enough_Metrics_Are_Available_Should_Vote_None(int metricCount)
{
var metrics = new List <KafkaTriggerMetrics>();
for (int i = 0; i < metricCount; i++)
{
metrics.Add(new KafkaTriggerMetrics(i, partitions.Count));
}
var context = new ScaleStatusContext <KafkaTriggerMetrics>()
{
Metrics = metrics.ToArray(),
WorkerCount = 1,
};
var result = topicScaler.GetScaleStatus(context);
Assert.NotNull(result);
Assert.Equal(ScaleVote.None, result.Vote);
}
public void When_No_Lag_Is_Found_Should_Vote_Scale_Down()
{
var context = new ScaleStatusContext <KafkaTriggerMetrics>()
{
Metrics = new KafkaTriggerMetrics[]
{
new KafkaTriggerMetrics(0, partitions.Count),
new KafkaTriggerMetrics(0, partitions.Count),
new KafkaTriggerMetrics(0, partitions.Count),
new KafkaTriggerMetrics(0, partitions.Count),
new KafkaTriggerMetrics(0, partitions.Count),
},
WorkerCount = 1,
};
var result = topicScaler.GetScaleStatus(context);
Assert.NotNull(result);
Assert.Equal(ScaleVote.ScaleIn, result.Vote);
}
public void GetScaleStatus_UnderSampleCountThreshold_ReturnsVote_None()
{
var context = new ScaleStatusContext <RabbitMQTriggerMetrics>
{
WorkerCount = 1
};
context.Metrics = new List <RabbitMQTriggerMetrics>
{
new RabbitMQTriggerMetrics {
QueueLength = 5, Timestamp = DateTime.UtcNow
},
new RabbitMQTriggerMetrics {
QueueLength = 10, Timestamp = DateTime.UtcNow
}
};
var status = _testListener.GetScaleStatus(context);
Assert.Equal(ScaleVote.None, status.Vote);
}
public void GetScaleStatus_EventHubSteady_ReturnsVote_ScaleIn()
{
var context = new ScaleStatusContext <EventHubsTriggerMetrics>
{
WorkerCount = 2
};
var timestamp = DateTime.UtcNow;
context.Metrics = new List <EventHubsTriggerMetrics>
{
new EventHubsTriggerMetrics {
EventCount = 1500, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 1600, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 1400, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 1300, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 1700, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 1600, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
};
var status = _scaleMonitor.GetScaleStatus(context);
Assert.AreEqual(ScaleVote.None, status.Vote);
var logs = _loggerProvider.GetAllLogMessages().ToArray();
var log = logs[0];
Assert.AreEqual(LogLevel.Information, log.Level);
Assert.AreEqual($"EventHubs entity '{_eventHubName}' is steady.", log.FormattedMessage);
}
public void GetScaleStatus_EventCountDecreasing_ReturnsVote_ScaleOut()
{
var context = new ScaleStatusContext <EventHubsTriggerMetrics>
{
WorkerCount = 1
};
var timestamp = DateTime.UtcNow;
context.Metrics = new List <EventHubsTriggerMetrics>
{
new EventHubsTriggerMetrics {
EventCount = 150, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 100, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 80, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 40, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 20, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 10, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
};
var status = _scaleMonitor.GetScaleStatus(context);
Assert.AreEqual(ScaleVote.ScaleIn, status.Vote);
var logs = _loggerProvider.GetAllLogMessages().ToArray();
var log = logs[0];
Assert.AreEqual(LogLevel.Information, log.Level);
Assert.AreEqual($"Event count is decreasing for '{_eventHubName}'.", log.FormattedMessage);
}
public void GetScaleStatus_QueueIdle_ReturnsVote_ScaleIn()
{
var context = new ScaleStatusContext <ServiceBusTriggerMetrics>
{
WorkerCount = 3
};
var timestamp = DateTime.UtcNow;
context.Metrics = new List <ServiceBusTriggerMetrics>
{
new ServiceBusTriggerMetrics {
MessageCount = 0, PartitionCount = 0, QueueTime = TimeSpan.Zero, Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 0, PartitionCount = 0, QueueTime = TimeSpan.Zero, Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 0, PartitionCount = 0, QueueTime = TimeSpan.Zero, Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 0, PartitionCount = 0, QueueTime = TimeSpan.Zero, Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 0, PartitionCount = 0, QueueTime = TimeSpan.Zero, Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 0, PartitionCount = 0, QueueTime = TimeSpan.Zero, Timestamp = timestamp.AddSeconds(15)
},
};
var status = _scaleMonitor.GetScaleStatus(context);
Assert.AreEqual(ScaleVote.ScaleIn, status.Vote);
var logs = _loggerProvider.GetAllLogMessages().ToArray();
var log = logs[0];
Assert.AreEqual(LogLevel.Information, log.Level);
Assert.AreEqual($"'{_entityPath}' is idle.", log.FormattedMessage);
}
public void GetScaleStatus_QueueSteady_ReturnsVote_None()
{
var context = new ScaleStatusContext <ServiceBusTriggerMetrics>
{
WorkerCount = 2
};
var timestamp = DateTime.UtcNow;
context.Metrics = new List <ServiceBusTriggerMetrics>
{
new ServiceBusTriggerMetrics {
MessageCount = 1500, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(1), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 1600, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(1), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 1400, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(1), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 1300, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(1), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 1700, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(1), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 1600, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(1), Timestamp = timestamp.AddSeconds(15)
},
};
var status = _scaleMonitor.GetScaleStatus(context);
Assert.AreEqual(ScaleVote.None, status.Vote);
var logs = _loggerProvider.GetAllLogMessages().ToArray();
var log = logs[0];
Assert.AreEqual(LogLevel.Information, log.Level);
Assert.AreEqual($"Service Bus entity '{_entityPath}' is steady.", log.FormattedMessage);
}
public void GetScaleStatus_QueueTimeIncreasing_ReturnsVote_ScaleOut()
{
var context = new ScaleStatusContext <ServiceBusTriggerMetrics>
{
WorkerCount = 1
};
var timestamp = DateTime.UtcNow;
context.Metrics = new List <ServiceBusTriggerMetrics>
{
new ServiceBusTriggerMetrics {
MessageCount = 100, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(1), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 100, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(2), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 100, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(3), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 100, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(4), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 100, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(5), Timestamp = timestamp.AddSeconds(15)
},
new ServiceBusTriggerMetrics {
MessageCount = 100, PartitionCount = 0, QueueTime = TimeSpan.FromSeconds(6), Timestamp = timestamp.AddSeconds(15)
},
};
var status = _scaleMonitor.GetScaleStatus(context);
Assert.AreEqual(ScaleVote.ScaleOut, status.Vote);
var logs = _loggerProvider.GetAllLogMessages().ToArray();
var log = logs[0];
Assert.AreEqual(LogLevel.Information, log.Level);
Assert.AreEqual($"Queue time is increasing for '{_entityPath}'.", log.FormattedMessage);
}
public void GetScaleStatus_EventHubIdle_ReturnsVote_ScaleIn()
{
var context = new ScaleStatusContext <EventHubsTriggerMetrics>
{
WorkerCount = 3
};
var timestamp = DateTime.UtcNow;
context.Metrics = new List <EventHubsTriggerMetrics>
{
new EventHubsTriggerMetrics {
EventCount = 0, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 0, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 0, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 0, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 0, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
new EventHubsTriggerMetrics {
EventCount = 0, PartitionCount = 0, Timestamp = timestamp.AddSeconds(15)
},
};
var status = _scaleMonitor.GetScaleStatus(context);
Assert.AreEqual(ScaleVote.ScaleIn, status.Vote);
var logs = _loggerProvider.GetAllLogMessages().ToArray();
var log = logs[0];
Assert.AreEqual(Extensions.Logging.LogLevel.Information, log.Level);
Assert.AreEqual($"'{_eventHubName}' is idle.", log.FormattedMessage);
}
