internal async Task WriteMetricsAsync(IScaleMonitor monitor, IEnumerable <ScaleMetrics> metrics, DateTime?now = null) { var batch = new TableBatchOperation(); await AccumulateMetricsBatchAsync(batch, monitor, metrics, now); await ExecuteBatchSafeAsync(batch, now); }
public void GetMonitor_ReturnsExpectedValue() { var functionId = "FunctionId"; var eventHubName = "EventHubName"; var consumerGroup = "ConsumerGroup"; var storageUri = new Uri("https://eventhubsteststorageaccount.blob.core.windows.net/"); var testLogger = new TestLogger("Test"); var listener = new EventHubListener( functionId, eventHubName, consumerGroup, "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=abc123=", "DefaultEndpointsProtocol=https;AccountName=EventHubScaleMonitorFakeTestAccount;AccountKey=ABCDEFG;EndpointSuffix=core.windows.net", new Mock <ITriggeredFunctionExecutor>(MockBehavior.Strict).Object, null, false, new EventHubOptions(), testLogger, new Mock <CloudBlobContainer>(MockBehavior.Strict, new Uri("https://eventhubsteststorageaccount.blob.core.windows.net/azure-webjobs-eventhub")).Object); IScaleMonitor scaleMonitor = listener.GetMonitor(); Assert.Equal(typeof(EventHubsScaleMonitor), scaleMonitor.GetType()); Assert.Equal($"{functionId}-EventHubTrigger-{eventHubName}-{consumerGroup}".ToLower(), scaleMonitor.Descriptor.Id); var scaleMonitor2 = listener.GetMonitor(); Assert.Same(scaleMonitor, scaleMonitor2); }
public void GetMonitor_ReturnsExpectedValue() { var functionId = "FunctionId"; var eventHubName = "EventHubName"; var consumerGroup = "ConsumerGroup"; var host = new EventProcessorHost(consumerGroup, "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=abc123=", eventHubName, new EventProcessorOptions(), 3, null); var consumerClientMock = new Mock <IEventHubConsumerClient>(); consumerClientMock.SetupGet(c => c.ConsumerGroup).Returns(consumerGroup); consumerClientMock.SetupGet(c => c.EventHubName).Returns(eventHubName); var listener = new EventHubListener( functionId, Mock.Of <ITriggeredFunctionExecutor>(), host, false, consumerClientMock.Object, Mock.Of <BlobsCheckpointStore>(), new EventHubOptions(), Mock.Of <LoggerFactory>()); IScaleMonitor scaleMonitor = listener.GetMonitor(); Assert.AreEqual(typeof(EventHubsScaleMonitor), scaleMonitor.GetType()); Assert.AreEqual($"{functionId}-EventHubTrigger-{eventHubName}-{consumerGroup}".ToLower(), scaleMonitor.Descriptor.Id); var scaleMonitor2 = listener.GetMonitor(); Assert.AreSame(scaleMonitor, scaleMonitor2); }
public async Task ReadMetricsAsync_NoMetricsForMonitor_ReturnsEmpty() { var monitor1 = new TestScaleMonitor1(); var monitors = new IScaleMonitor[] { monitor1 }; var result = await _repository.ReadMetricsAsync(monitors); Assert.Equal(1, result.Count); Assert.Empty(result[monitor1]); }
public override bool TryGetScaleMonitor( string functionId, string functionName, string hubName, string storageConnectionString, out IScaleMonitor scaleMonitor) { scaleMonitor = this.scaleMonitor ??= new SqlScaleMonitor(this.service, hubName); return(true); }
/// <summary> /// Tries to obtain a scale monitor for autoscaling. /// </summary> /// <param name="functionId">Function id.</param> /// <param name="functionName">Function name.</param> /// <param name="hubName">Task hub name.</param> /// <param name="storageConnectionString">Storage account connection string, used for Azure Storage provider.</param> /// <param name="scaleMonitor">The scale monitor.</param> /// <returns>True if autoscaling is supported, false otherwise.</returns> public virtual bool TryGetScaleMonitor( string functionId, string functionName, string hubName, string storageConnectionString, out IScaleMonitor scaleMonitor) { scaleMonitor = null; return(false); }
public void GetMonitor_ReturnsExpectedValue() { IScaleMonitor scaleMonitor = this.listener.GetMonitor(); Assert.Equal(typeof(DurableTaskScaleMonitor), scaleMonitor.GetType()); Assert.Equal($"{this.functionId}-DurableTaskTrigger-DurableTaskHub".ToLower(), scaleMonitor.Descriptor.Id); var scaleMonitor2 = this.listener.GetMonitor(); Assert.Same(scaleMonitor, scaleMonitor2); }
public void GetMonitor_ReturnsExpectedValue() { IScaleMonitor scaleMonitor = _listener.GetMonitor(); Assert.AreEqual(typeof(ServiceBusScaleMonitor), scaleMonitor.GetType()); Assert.AreEqual(scaleMonitor.Descriptor.Id, $"{_functionId}-ServiceBusTrigger-{_entityPath}".ToLower()); var scaleMonitor2 = _listener.GetMonitor(); Assert.AreSame(scaleMonitor, scaleMonitor2); }
public async Task TableRead_ManyRows_Succeeds() { var monitor1 = new TestScaleMonitor1(); var monitors = new IScaleMonitor[] { monitor1 }; var metricsTable = _repository.GetMetricsTable(); await _repository.CreateIfNotExistsAsync(metricsTable); TableBatchOperation batch = new TableBatchOperation(); int numRows = 500; for (int i = 0; i < numRows; i++) { var sample = new TestScaleMetrics1 { Count = i }; await Task.Delay(5); batch.Add(TableStorageScaleMetricsRepository.CreateMetricsInsertOperation(sample, TestHostId, monitor1.Descriptor)); if (batch.Count % 100 == 0) { await metricsTable.ExecuteBatchAsync(batch); batch = new TableBatchOperation(); } } if (batch.Count > 0) { await metricsTable.ExecuteBatchAsync(batch); } var results = await _repository.ReadMetricsAsync(monitors); Assert.Equal(1, results.Count); Assert.Equal(numRows, results[monitor1].Count); // verify results are returned in the order they were inserted (ascending // time order) and the timestamps are monotonically increasing var metrics = results[monitor1].ToArray(); for (int i = 0; i < numRows - 1; i++) { for (int j = i + 1; j < numRows; j++) { var m1 = (TestScaleMetrics1)metrics[i]; var m2 = (TestScaleMetrics1)metrics[j]; Assert.True(m1.Count < m2.Count); Assert.True(metrics[i].Timestamp < metrics[j].Timestamp); } } }
public async Task WriteAsync_RollsToNewTable() { // delete any existing non-current metrics tables var tables = await _repository.ListOldMetricsTablesAsync(); foreach (var table in tables) { await table.DeleteIfExistsAsync(); } // set now to months back int numTables = 3; var startDate = DateTime.UtcNow.AddMonths(-1 * numTables); // now start writing metrics in each old month DateTime now = startDate; var monitor1 = new TestScaleMonitor1(); var monitors = new IScaleMonitor[] { monitor1 }; for (int i = 0; i < numTables; i++) { var table = _repository.GetMetricsTable(now); var monitorMetrics = new List <ScaleMetrics>(); for (int j = 0; j < 10; j++) { monitorMetrics.Add(new TestScaleMetrics1 { Count = j }); } await _repository.WriteMetricsAsync(monitor1, monitorMetrics, now); now = now.AddMonths(1); } // verify 3 tables were created tables = await _repository.ListOldMetricsTablesAsync(); Assert.Equal(3, tables.Count()); // make sure the current metrics table isn't in the list var currTable = _repository.GetMetricsTable(); Assert.False(tables.Select(p => p.Name).Contains(currTable.Name)); // delete the tables await _repository.DeleteOldMetricsTablesAsync(); tables = await _repository.ListOldMetricsTablesAsync(); Assert.Empty(tables); }
internal Type GetMonitorScaleMetricsTypeOrNull(IScaleMonitor monitor) { var monitorInterfaceType = monitor.GetType().GetInterfaces().SingleOrDefault(p => p.IsGenericType && p.GetGenericTypeDefinition() == typeof(IScaleMonitor <>)); if (monitorInterfaceType != null) { return(monitorInterfaceType.GetGenericArguments()[0]); } // we require the monitor to implement the generic interface in order to know // what type to deserialize into _logger.LogWarning($"Monitor {monitor.GetType().FullName} doesn't implement {typeof(IScaleMonitor<>)}."); return(null); }
public async Task ReadMetricsAsync_InvalidMonitor_ReturnsEmpty() { var monitor1 = new TestInvalidScaleMonitor(); var monitors = new IScaleMonitor[] { monitor1 }; var result = await _repository.ReadMetricsAsync(monitors); Assert.Equal(1, result.Count); Assert.Empty(result[monitor1]); var log = _loggerProvider.GetAllLogMessages().Single(); Assert.Equal("Monitor Microsoft.Azure.WebJobs.Script.Tests.TestInvalidScaleMonitor doesn't implement Microsoft.Azure.WebJobs.Host.Scale.IScaleMonitor`1[TMetrics].", log.FormattedMessage); }
/// <inheritdoc/> public override bool TryGetScaleMonitor( string functionId, string functionName, string hubName, string storageConnectionString, out IScaleMonitor scaleMonitor) { scaleMonitor = new DurableTaskScaleMonitor( functionId, functionName, hubName, storageConnectionString, this.logger); return(true); }
internal async Task AccumulateMetricsBatchAsync(TableBatchOperation batch, IScaleMonitor monitor, IEnumerable <ScaleMetrics> metrics, DateTime?now = null) { if (!metrics.Any()) { return; } string hostId = await _hostIdProvider.GetHostIdAsync(CancellationToken.None); foreach (var sample in metrics) { var operation = CreateMetricsInsertOperation(sample, hostId, monitor.Descriptor, now); batch.Add(operation); } }
public async Task InvalidStorageConnection_Handled() { var configuration = new ConfigurationBuilder().Build(); Assert.Null(configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage)); var options = new ScaleOptions(); ILoggerFactory loggerFactory = new LoggerFactory(); loggerFactory.AddProvider(_loggerProvider); var localRepository = new TableStorageScaleMetricsRepository(configuration, _hostIdProviderMock.Object, new OptionsWrapper <ScaleOptions>(options), loggerFactory, new TestEnvironment()); var monitor1 = new TestScaleMonitor1(); var monitor2 = new TestScaleMonitor2(); var monitor3 = new TestScaleMonitor3(); var monitors = new IScaleMonitor[] { monitor1, monitor2, monitor3 }; var result = await localRepository.ReadMetricsAsync(monitors); Assert.Empty(result); var logs = _loggerProvider.GetAllLogMessages(); Assert.Single(logs); Assert.Equal("Azure Storage connection string is empty or invalid. Unable to read/write scale metrics.", logs[0].FormattedMessage); _loggerProvider.ClearAllLogMessages(); Dictionary <IScaleMonitor, ScaleMetrics> metricsMap = new Dictionary <IScaleMonitor, ScaleMetrics>(); metricsMap.Add(monitor1, new TestScaleMetrics1 { Count = 10 }); metricsMap.Add(monitor2, new TestScaleMetrics2 { Num = 50 }); metricsMap.Add(monitor3, new TestScaleMetrics3 { Length = 100 }); await localRepository.WriteMetricsAsync(metricsMap); }
public async Task ReadMetricsAsync_FiltersExpiredMetrics() { var monitor1 = new TestScaleMonitor1(); var monitors = new IScaleMonitor[] { monitor1 }; // add a bunch of expired samples var batch = new TableBatchOperation(); for (int i = 5; i > 0; i--) { var metrics = new TestScaleMetrics1 { Count = i }; var now = DateTime.UtcNow - _scaleOptions.ScaleMetricsMaxAge - TimeSpan.FromMinutes(i); await _repository.AccumulateMetricsBatchAsync(batch, monitor1, new ScaleMetrics[] { metrics }, now); } // add a few samples that aren't expired for (int i = 3; i > 0; i--) { var metrics = new TestScaleMetrics1 { Count = 77 }; var now = DateTime.UtcNow - TimeSpan.FromSeconds(i); await _repository.AccumulateMetricsBatchAsync(batch, monitor1, new ScaleMetrics[] { metrics }, now); } await _repository.ExecuteBatchSafeAsync(batch); var result = await _repository.ReadMetricsAsync(monitors); var resultMetrics = result[monitor1].Cast <TestScaleMetrics1>().ToArray(); Assert.Equal(3, resultMetrics.Length); Assert.All(resultMetrics, p => Assert.Equal(77, p.Count)); }
public async Task ReadWriteMetrics_IntegerConversion_HandlesLongs() { var monitor1 = new TestScaleMonitor1(); var monitors = new IScaleMonitor[] { monitor1 }; // first write a couple entities manually to the table to simulate // the change in entity property type (int -> long) // this shows that the table can have entities of both formats with // no versioning issues // add an entity with Count property of type int var entity = new DynamicTableEntity { RowKey = TableStorageScaleMetricsRepository.GetRowKey(DateTime.UtcNow), PartitionKey = TestHostId, Properties = new Dictionary <string, EntityProperty>() }; var expectedIntCountValue = int.MaxValue; entity.Properties.Add("Timestamp", new EntityProperty(DateTime.UtcNow)); entity.Properties.Add("Count", new EntityProperty(expectedIntCountValue)); entity.Properties.Add(TableStorageScaleMetricsRepository.MonitorIdPropertyName, EntityProperty.GeneratePropertyForString(monitor1.Descriptor.Id)); var batch = new TableBatchOperation(); batch.Add(TableOperation.Insert(entity)); // add an entity with Count property of type long entity = new DynamicTableEntity { RowKey = TableStorageScaleMetricsRepository.GetRowKey(DateTime.UtcNow), PartitionKey = TestHostId, Properties = new Dictionary <string, EntityProperty>() }; var expectedLongCountValue = long.MaxValue; entity.Properties.Add("Timestamp", new EntityProperty(DateTime.UtcNow)); entity.Properties.Add("Count", new EntityProperty(expectedLongCountValue)); entity.Properties.Add(TableStorageScaleMetricsRepository.MonitorIdPropertyName, EntityProperty.GeneratePropertyForString(monitor1.Descriptor.Id)); batch.Add(TableOperation.Insert(entity)); await _repository.ExecuteBatchSafeAsync(batch); // push a long max value through serialization var metricsMap = new Dictionary <IScaleMonitor, ScaleMetrics>(); metricsMap.Add(monitor1, new TestScaleMetrics1 { Count = long.MaxValue }); await _repository.WriteMetricsAsync(metricsMap); // add one more metricsMap = new Dictionary <IScaleMonitor, ScaleMetrics>(); metricsMap.Add(monitor1, new TestScaleMetrics1 { Count = 12345 }); await _repository.WriteMetricsAsync(metricsMap); // read the metrics back var result = await _repository.ReadMetricsAsync(monitors); Assert.Equal(1, result.Count); var monitorMetricsList = result[monitor1]; Assert.Equal(4, monitorMetricsList.Count); // verify the explicitly written int record was read correctly var currSample = (TestScaleMetrics1)monitorMetricsList[0]; Assert.Equal(expectedIntCountValue, currSample.Count); // verify the explicitly written long record was read correctly currSample = (TestScaleMetrics1)monitorMetricsList[1]; Assert.Equal(expectedLongCountValue, currSample.Count); // verify the final roundtripped values currSample = (TestScaleMetrics1)monitorMetricsList[2]; Assert.Equal(long.MaxValue, currSample.Count); currSample = (TestScaleMetrics1)monitorMetricsList[3]; Assert.Equal(12345, currSample.Count); }
public async Task WriteMetricsAsync_PersistsMetrics() { var monitor1 = new TestScaleMonitor1(); var monitor2 = new TestScaleMonitor2(); var monitor3 = new TestScaleMonitor3(); var monitors = new IScaleMonitor[] { monitor1, monitor2, monitor3 }; var result = await _repository.ReadMetricsAsync(monitors); Assert.Equal(3, result.Count); // simulate 10 sample iterations for (int i = 0; i < 10; i++) { Dictionary <IScaleMonitor, ScaleMetrics> metricsMap = new Dictionary <IScaleMonitor, ScaleMetrics>(); metricsMap.Add(monitor1, new TestScaleMetrics1 { Count = i }); metricsMap.Add(monitor2, new TestScaleMetrics2 { Num = i }); metricsMap.Add(monitor3, new TestScaleMetrics3 { Length = i }); await _repository.WriteMetricsAsync(metricsMap); } // read the metrics back result = await _repository.ReadMetricsAsync(monitors); Assert.Equal(3, result.Count); var monitorMetricsList = result[monitor1]; for (int i = 0; i < 10; i++) { var currSample = (TestScaleMetrics1)monitorMetricsList[i]; Assert.Equal(i, currSample.Count); Assert.NotEqual(default(DateTime), currSample.Timestamp); } monitorMetricsList = result[monitor2]; for (int i = 0; i < 10; i++) { var currSample = (TestScaleMetrics2)monitorMetricsList[i]; Assert.Equal(i, currSample.Num); Assert.NotEqual(default(DateTime), currSample.Timestamp); } monitorMetricsList = result[monitor3]; for (int i = 0; i < 10; i++) { var currSample = (TestScaleMetrics3)monitorMetricsList[i]; Assert.Equal(i, currSample.Length); Assert.NotEqual(default(DateTime), currSample.Timestamp); } // if no monitors are presented result will be empty monitors = new IScaleMonitor[0]; result = await _repository.ReadMetricsAsync(monitors); Assert.Equal(0, result.Count); }