public void ListDcrBySubscriptionTest()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
string[] dcrNames = { "dcrListOneTest", "dcrListTwoTest", "dcrListThreeTest", "dcrListFourTest" };;
MonitorManagementClient insightsClient = GetMonitorManagementClient(context, handler);
foreach (var dcrName in dcrNames)
{
_ = CreateDcr(insightsClient, dcrName);
}
var dcrList = insightsClient.DataCollectionRules.ListBySubscription().ToList();
Assert.NotNull(dcrList);
Assert.True(dcrList.Count >= 3, "List must be greather than 3 items");
foreach (var dcrName in dcrNames)
{
Assert.Equal(1, dcrList.Count(x => x.Name == dcrName));
}
foreach (var dcrName in dcrNames)
{
insightsClient.DataCollectionRules.Delete(ResourceGroupName, dcrName);
}
}
}
private void GetMultiResResourceGroupLevelMetricAlertRule(MonitorManagementClient insightsClient)
{
MetricAlertResource expectedParameters = GetCreateOrUpdateMultiResResourceGroupLevelRuleParameters();
MetricAlertResource alertRule = insightsClient.MetricAlerts.Get(resourceGroupName: ResourceGroupName, ruleName: MultiResResourceGroupLevelRuleName);
Utilities.AreEqual(expectedParameters, alertRule);
}
public void DcrAssociationTest()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
var dcrName = "dcraSdkCreateTest";
var dcraName = "dcraSdkCreateTestAssoc";
MonitorManagementClient insightsClient = GetMonitorManagementClient(context, handler);
var dcr = CreateDcr(insightsClient, dcrName);
Assert.NotNull(dcr);
var dcra = insightsClient.DataCollectionRuleAssociations.Create(vmResourceUri, dcraName, new DataCollectionRuleAssociationProxyOnlyResource {
DataCollectionRuleId = dcr.Id,
Description = "Assoc with virtual machine"
});
Assert.NotNull(dcra);
var dcraListByRule = insightsClient.DataCollectionRuleAssociations.ListByRule(ResourceGroupName, dcrName);
Assert.Equal(1, dcraListByRule.Count(x => x.Name == dcraName.ToLower()));
var dcraListByResource = insightsClient.DataCollectionRuleAssociations.ListByResource(vmResourceUri);
Assert.Equal(1, dcraListByResource.Count(x => x.Name == dcraName));
var dcraFromGet = insightsClient.DataCollectionRuleAssociations.Get(vmResourceUri, dcraName);
AreEqual(dcra, dcraFromGet);
insightsClient.DataCollectionRuleAssociations.Delete(vmResourceUri, dcra.Name);
insightsClient.DataCollectionRules.Delete(ResourceGroupName, dcrName);
}
}
public void UpdateDcrTest()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
var dcrName = "dcrSdkUpdateTest";
MonitorManagementClient insightsClient = GetMonitorManagementClient(context, handler);
var dcr = CreateDcr(insightsClient, dcrName);
Assert.NotNull(dcr);
dcr = insightsClient.DataCollectionRules.Update(ResourceGroupName, dcrName, new ResourceForUpdate {
Tags = new Dictionary <string, string>
{
{ "TagUpdated", "ValueUpdate" }
}
});
Assert.NotNull(dcr);
var dcrList = insightsClient.DataCollectionRules.ListByResourceGroup(ResourceGroupName).ToList();
dcr = dcrList.FirstOrDefault(x => x.Name == dcrName);
Assert.NotNull(dcr);
Assert.True(dcr.Tags.Count == 1);
Assert.Equal("TagUpdated", dcr.Tags.Keys.First());
Assert.Equal("ValueUpdate", dcr.Tags["TagUpdated"]);
insightsClient.DataCollectionRules.Delete(ResourceGroupName, dcrName);
}
}
private async Task <MonitorManagementClient> GetAzureMonitorClientAsync()
{
var azureServiceCredentials = await AuthenticateWithAzureAsync();
var monitoringClient = new MonitorManagementClient(azureServiceCredentials);
return(monitoringClient);
}
protected override async Task ScrapeResourceAsync(MonitorManagementClient monitoringClient, ServiceBusQueueMetricDefinition metricDefinition)
{
var resourceUri = string.Format(ResourceUriTemplate, AzureMetadata.SubscriptionId, AzureMetadata.ResourceGroupName, metricDefinition.Namespace);
var foundMetric = await QueryAzureMonitorAsync(monitoringClient, resourceUri, metricDefinition.QueueName, metricDefinition.AzureMetricConfiguration.MetricName, metricDefinition.AzureMetricConfiguration.Aggregation);
var gauge = Metrics.CreateGauge(metricDefinition.Name, metricDefinition.Description);
gauge.Set(foundMetric);
}
private void CreateOrUpdateMetricAlertRule(MonitorManagementClient insightsClient)
{
MetricAlertResource expectedParameters = GetCreateOrUpdateRuleParameters();
MetricAlertResource result = insightsClient.MetricAlerts.CreateOrUpdate(
resourceGroupName: ResourceGroupName,
ruleName: RuleName,
parameters: expectedParameters);
Utilities.AreEqual(expectedParameters, result);
}
private DataCollectionRuleResource CreateDcr(MonitorManagementClient insightsClient, string dcrName)
{
return(insightsClient.DataCollectionRules.Create(
resourceGroupName: ResourceGroupName,
dataCollectionRuleName: dcrName,
new DataCollectionRuleResource
{
Location = "East US",
Tags = new Dictionary <string, string>
{
{ "tagOne", "valueOne" },
{ "tagTwo", "valueTwo" }
},
DataSources = new DataCollectionRuleDataSources
{
PerformanceCounters = new List <PerfCounterDataSource>
{
new PerfCounterDataSource
{
Name = "perfCounterDataSource1",
Streams = new List <string> {
"Microsoft-InsightsMetrics"
},
ScheduledTransferPeriod = "PT1M",
SamplingFrequencyInSeconds = 10,
CounterSpecifiers = new List <string>
{
"\\Memory\\% Committed Bytes In Use",
"\\Memory\\Available Bytes",
"\\Network Interface(*)\\Bytes Received/sec",
}
}
}
},
Destinations = new DataCollectionRuleDestinations
{
AzureMonitorMetrics = new DestinationsSpecAzureMonitorMetrics {
Name = "ammDestination"
}
},
DataFlows = new List <DataFlow>
{
new DataFlow
{
Streams = new List <string> {
"Microsoft-InsightsMetrics"
},
Destinations = new List <string> {
"ammDestination"
}
}
}
}
));
}
private static async Task <MonitorManagementClient> AuthenticateWithReadOnlyClient(string tenantId, string clientId, string secret, string subscriptionId)
{
// Build the service credentials and Monitor client
var serviceCreds = await ApplicationTokenProvider.LoginSilentAsync(tenantId, clientId, secret);
var monitorClient = new MonitorManagementClient(serviceCreds);
monitorClient.SubscriptionId = subscriptionId;
return(monitorClient);
}
private static void Authenticate(string tenantId, string clientId, string secret, string subscriptionId)
{
var creds = SdkContext.AzureCredentialsFactory.FromServicePrincipal(
clientId, secret, tenantId, AzureEnvironment.AzureGlobalCloud
);
_azure = Azure
.Authenticate(creds)
.WithSubscription(subscriptionId);
_monitor = new MonitorManagementClient(creds);
_monitor.SubscriptionId = subscriptionId;
}
private void MultiResSubscriptionLevelDynamicMetricAlertRule()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
MonitorManagementClient insightsClient = GetMonitorManagementClient(context, handler);
this.VerifyExistenceOrCreateResourceGroup(resourceGroupName: ResourceGroupName, location: Location);
CreateOrUpdateMultiResSubscriptionLevelDynamicMetricAlertRule(insightsClient);
GetMultiResSubscriptionLevelDynamicMetricAlertRule(insightsClient);
DeleteMultiResSubscriptionLevelDynamicMetricAlertRule(insightsClient);
}
}
public void LogProfiles_DeleteTest()
{
HttpResponseMessage response = new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(string.Empty)
};
var handler = new RecordedDelegatingHandler(response);
MonitorManagementClient customClient = this.GetMonitorManagementClient(handler);
customClient.LogProfiles.Delete(logProfileName: DefaultName);
}
public void MetricAlertRuleFlow()
{
using (MockContext context = MockContext.Start(this.GetType().FullName))
{
MonitorManagementClient insightsClient = GetMonitorManagementClient(context, handler);
this.VerifyExistenceOrCreateResourceGroup(resourceGroupName: ResourceGroupName, location: Location);
CreateOrUpdateMetricAlertRule(insightsClient);
GetMetricAlertRule(insightsClient);
DeleteMetricAlertRule(insightsClient);
}
}
protected MonitorManagementClient GetMonitorManagementClient(RecordedDelegatingHandler handler)
{
handler.IsPassThrough = false;
var tokenProvider = new StringTokenProvider("granted", "SimpleString");
var id = Guid.NewGuid().ToString();
var token = new TokenCredentials(tokenProvider: tokenProvider, tenantId: id, callerId: id);
var client = new MonitorManagementClient(token, handler);
token.InitializeServiceClient(client);
client.SubscriptionId = id;
return(client);
}
private static void Check(
MonitorManagementClient insightsClient,
OperationListResult operationListResult)
{
Assert.Equal(46, operationListResult.Value.Count);
Assert.Equal("Microsoft.Insights/Operations/Read", operationListResult.Value[0].Name);
Assert.Equal("Operations read", operationListResult.Value[0].Display.Operation);
Assert.Equal("Microsoft Monitoring Insights", operationListResult.Value[0].Display.Provider);
Assert.Equal("Operations", operationListResult.Value[0].Display.Resource);
Assert.Equal("Microsoft.Insights/Webtests/Read", operationListResult.Value[45].Name);
Assert.Equal("Webtest read", operationListResult.Value[45].Display.Operation);
Assert.Equal("Microsoft Monitoring Insights", operationListResult.Value[45].Display.Provider);
Assert.Equal("Web tests", operationListResult.Value[45].Display.Resource);
}
public void CreateDcrTest()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
var dcrName = "dcrSdkCreateTest";
MonitorManagementClient insightsClient = GetMonitorManagementClient(context, handler);
var dcr = CreateDcr(insightsClient, dcrName);
Assert.NotNull(dcr);
var dcrFromGet = insightsClient.DataCollectionRules.Get(ResourceGroupName, dcrName);
insightsClient.DataCollectionRules.Delete(ResourceGroupName, dcrName);
AreEqual(dcr, dcrFromGet);
}
}
static async Task RunSampleAsync()
{
var tenantId = ""; // AAD Tenant
var clientId = ""; // AAD Web App ID. Do not use a native app
var secret = ""; // Your generated secret
var resourceId = ""; // resourceId can be taken when you select the namespace you intend to use in the portal and copy the url. Then delete everything before "subscriptions" and after the namespace name.
string entityName = ""; // Queue or Topic name
string metricName = "ActiveMessages"; // Valid metrics "IncomingMessages,IncomingRequests,ActiveMessages,Messages,Size"
string aggregation = "Total"; // Valid aggregations: Total and Average
// Create new Metrics token and Management client.
var serviceCreds = await ApplicationTokenProvider.LoginSilentAsync(tenantId, clientId, secret);
MonitorManagementClient monitoringClient = new MonitorManagementClient(serviceCreds);
var metricDefinitions = monitoringClient.MetricDefinitions.List(resourceId);
if (metricDefinitions.FirstOrDefault(
metric => string.Equals(metric.Name.Value, metricName, StringComparison.InvariantCultureIgnoreCase)) == null)
{
Console.WriteLine("Invalid metric");
return;
}
string startDate = DateTime.Now.AddHours(-1).ToString("o");
string endDate = DateTime.Now.ToString("o");
string timeSpan = startDate + "/" + endDate;
ODataQuery <MetadataValue> odataFilterMetrics = new ODataQuery <MetadataValue>($"EntityName eq '{entityName}'");
// Use this as quick and easy way to understand what metrics are emitted and what to query for.
// When looking for the count and size of an entity the only supported way is using total and 1 minute time slices.
// Accessing those metrics via code is mostly for auto scaling purposes on sender and receiver side.
Response metrics1 = monitoringClient.Metrics.List(resourceUri: resourceId, metricnames: metricName, odataQuery: odataFilterMetrics, timespan: timeSpan, aggregation: aggregation, interval: TimeSpan.FromMinutes(1));
Console.WriteLine(JsonConvert.SerializeObject(metrics1, Newtonsoft.Json.Formatting.Indented));
// Use this to get a list output to your console
var metrics = monitoringClient.Metrics.List(resourceId, odataFilterMetrics, timeSpan, TimeSpan.FromMinutes(1), metricName, aggregation);
EnumerateMetrics(metrics, resourceId, entityName);
Console.WriteLine("Press any key to exit.");
Console.ReadLine();
}
public void DeleteDcrTest()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
var dcrName = "dcrSdkDeleteTest";
MonitorManagementClient insightsClient = GetMonitorManagementClient(context, handler);
var dcrDelete = CreateDcr(insightsClient, dcrName);;
Assert.NotNull(dcrDelete);
var dcrList = insightsClient.DataCollectionRules.ListByResourceGroup(ResourceGroupName).ToList();
Assert.Equal(1, dcrList.Count(x => x.Name == dcrName));
insightsClient.DataCollectionRules.Delete(ResourceGroupName, dcrName);
dcrList = insightsClient.DataCollectionRules.ListByResourceGroup(ResourceGroupName).ToList();
Assert.Equal(0, dcrList.Count(x => x.Name == dcrName));
}
}
public void ListGetIncidentsTest()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
MonitorManagementClient insightsClient = GetMonitorManagementClient(context, handler);
// NOTE: for this test to work the alert rule must exist
var actualIncidents = insightsClient.AlertRuleIncidents.ListByAlertRule(
resourceGroupName: ResourceGroupName,
ruleName: RuleName);
Assert.NotNull(actualIncidents);
List <Incident> incidentsList = actualIncidents.ToList();
if (!this.IsRecording)
{
Assert.True(incidentsList.Count > 0, "List of incidents should not be 0 length.");
Assert.Equal(string.Format(provider: CultureInfo.InvariantCulture, format: ResourceId, args: insightsClient.SubscriptionId), incidentsList[0].RuleName, ignoreCase: true);
}
if (incidentsList.Count > 0)
{
string incidentName = incidentsList[0].Name;
var actualIncident = insightsClient.AlertRuleIncidents.Get(
resourceGroupName: ResourceGroupName,
ruleName: RuleName,
incidentName: incidentName);
if (!this.IsRecording)
{
Utilities.AreEqual(incidentsList[0], actualIncident);
}
}
}
}
public void MetricBasedRule()
{
using (MockContext context = MockContext.Start(this.GetType()))
{
MonitorManagementClient insightsClient = GetMonitorManagementClient(context, handler);
this.VerifyExistenceOrCreateResourceGroup(resourceGroupName: ResourceGroupName, location: Location);
AlertRuleResource expectedParameters = GetCreateOrUpdateRuleParameter(insightsClient.SubscriptionId);
AlertRuleResource result = insightsClient.AlertRules.CreateOrUpdate(
resourceGroupName: ResourceGroupName,
ruleName: RuleName,
parameters: expectedParameters);
if (!this.IsRecording)
{
Check(result);
}
AlertRuleResource retrievedRule = insightsClient.AlertRules.Get(
resourceGroupName: ResourceGroupName,
ruleName: RuleName);
if (!this.IsRecording)
{
Check(retrievedRule);
Utilities.AreEqual(result, retrievedRule);
}
IEnumerable <AlertRuleResource> enumOfRules = insightsClient.AlertRules.ListByResourceGroup(
resourceGroupName: ResourceGroupName);
if (!this.IsRecording)
{
var listOfRules = enumOfRules.ToList();
var selected = listOfRules.Where(r => string.Equals(r.Id, retrievedRule.Id, StringComparison.OrdinalIgnoreCase)).ToList();
Assert.NotNull(selected);
Assert.NotEmpty(selected);
Assert.True(selected.Count == 1);
Utilities.AreEqual(retrievedRule, selected[0]);
}
var newTags = new Dictionary <string, string>()
{
{ "key2", "val2" }
};
// TODO: Update is requiring 'location', but it was not specified so.
AlertRuleResourcePatch pathResource = new AlertRuleResourcePatch(
name: retrievedRule.Name,
isEnabled: !retrievedRule.IsEnabled,
tags: newTags,
actions: retrievedRule.Actions,
condition: retrievedRule.Condition,
description: retrievedRule.Description,
lastUpdatedTime: retrievedRule.LastUpdatedTime
);
AlertRuleResource updatedRule = null;
Assert.Throws <ErrorResponseException>(
() => updatedRule = insightsClient.AlertRules.Update(
resourceGroupName: ResourceGroupName,
ruleName: RuleName,
alertRulesResource: pathResource));
if (!this.IsRecording && updatedRule != null)
{
Check(updatedRule);
Assert.NotEqual(retrievedRule.Tags, updatedRule.Tags);
Assert.True(retrievedRule.IsEnabled = !updatedRule.IsEnabled);
Assert.Equal(retrievedRule.Name, updatedRule.Name);
Assert.Equal(retrievedRule.Location, updatedRule.Location);
Assert.Equal(retrievedRule.Id, updatedRule.Id);
}
AlertRuleResource retrievedUpdatedRule = insightsClient.AlertRules.Get(
resourceGroupName: ResourceGroupName,
ruleName: RuleName);
if (!this.IsRecording && updatedRule != null)
{
Check(retrievedRule);
Utilities.AreEqual(updatedRule, retrievedUpdatedRule);
}
insightsClient.AlertRules.Delete(
resourceGroupName: ResourceGroupName,
ruleName: RuleName);
Assert.Throws <ErrorResponseException>(
() => insightsClient.AlertRules.Get(
resourceGroupName: ResourceGroupName,
ruleName: RuleName));
}
}
/// <summary> /// Scrapes the configured resource /// </summary> /// <param name="monitoringClient">Client to query Azure Monitor</param> /// <param name="metricDefinition">Definition of the metric to scrape</param> protected abstract Task ScrapeResourceAsync(MonitorManagementClient monitoringClient, TMetricDefinition metricDefinition);
//Runs Every 2 minutes looking for changes to network settings across the subscription
public static void Run([TimerTrigger("0 */2 * * * *")] TimerInfo myTimer, TraceWriter log)
{
log.Info($"NSGRuleMonitor Timer trigger function executed at: {DateTime.Now}");
//Storage Account Table Access for state information (i.e. LastRun time)
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(System.Environment.GetEnvironmentVariable("AZURE_STORAGE"));
tableClient = storageAccount.CreateCloudTableClient();
//Configuration for connections/operations on Azure Resources
var tenantId = System.Environment.GetEnvironmentVariable("AZURE_TENANT_ID");
var clientId = System.Environment.GetEnvironmentVariable("AZURE_CLIENT_ID");
var secret = System.Environment.GetEnvironmentVariable("AZURE_CLIENT_SECRET");
var subscriptionId = System.Environment.GetEnvironmentVariable("AZURE_SUBSCRIPTION_ID");
var storage = System.Environment.GetEnvironmentVariable("AZURE_STORAGE");
if (new List <string> {
tenantId, clientId, secret, subscriptionId
}.Any(i => String.IsNullOrEmpty(i)))
{
log.Error("Please provide environment variables for AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_SUBSCRIPTION_ID.");
}
else
{
//Get Management Client Access to Azure for Activity Monitor and Network
readOnlyClient = AuthenticateWithReadOnlyClient(tenantId, clientId, secret, subscriptionId).Result;
networkClient = AuthenticateWithNetworkClient(tenantId, clientId, secret, subscriptionId).Result;
//When did we last check for Network Security Compliance?
StateInfo lastrunstate = GetOrUpdateLastRunStateInfo().Result;
StateInfo currentstate = new StateInfo(DateTime.UtcNow.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'Z'"));
log.Info("NetworkResourceWatcher: Last Ran @" + lastrunstate.LastRunTimeUTC);
log.Info("NetworkResourceWatcher: Current Time @" + currentstate.LastRunTimeUTC);
//Start where we left off loading activity events looking for modifications accross the subscription for network resources
var filter = new ODataQuery <EventData>("eventTimestamp ge '" + lastrunstate.LastRunTimeUTC + "' and eventTimestamp le '" + currentstate.LastRunTimeUTC + "' and resourceProvider eq 'Microsoft.Network'");
var result = readOnlyClient.ActivityLogs.List(filter);
//Iterate over events looking for NSG security rule updates
List <FailedNSGData> nsgfails = new List <FailedNSGData>();
foreach (EventData d in result)
{
//Avoid duplicates if notification is all set to be sent for operationId
if (nsgfails.Any(prod => prod.OperationId == d.OperationId))
{
continue;
}
if (d.OperationName.Value.Contains("Microsoft.Network/networkSecurityGroups/securityRules/write"))
{
var split = d.ResourceId.Split('/');
var nsgname = split[8];
var rulename = split[10];
//Retrive the NSG details
NetworkSecurityGroup nsg = null;
try
{
nsg = networkClient.NetworkSecurityGroups.Get(d.ResourceGroupName, nsgname);
}
catch (Exception e)
{
log.Error("Error Fetching NetworkSecurityGroup: " + e.Message, e);
continue;
}
//Look for compliance failures that our Validator finds on the update/add of this NSG
var failures = new List <string>();
foreach (SecurityRule rule in nsg.SecurityRules)
{
if (!NSGRuleValidator.Check(rule, failures))
{
//This NSG update has failed compliance add it to our list of non-compliance NSGs
FailedNSGData data = new FailedNSGData()
{
OperationName = d.OperationName.Value, OperationId = d.OperationId, Subscription = d.SubscriptionId, ResourceGroup = d.ResourceGroupName, Caller = d.Caller, ResourceId = d.ResourceId, Timestamp = d.EventTimestamp.ToString(), ValidationFailures = failures.ToArray(), NetworkSecuirtyGroupName = nsgname, RuleName = rulename
};
nsgfails.Add(data);
}
}
}
else if (d.OperationName.Value.Contains("Microsoft.Network/networkSecurityGroups/securityRules/delete"))
{
//Deletion of any NSG causes a notification
var split = d.ResourceId.Split('/');
var nsgname = split[8];
var rulename = split[10];
string[] failures = { "NSGSecurityRuleWasDeleted" };
FailedNSGData data = new FailedNSGData()
{
OperationName = d.OperationName.Value, OperationId = d.OperationId, Subscription = d.SubscriptionId, ResourceGroup = d.ResourceGroupName, Caller = d.Caller, ResourceId = d.ResourceId, Timestamp = d.EventTimestamp.ToString(), ValidationFailures = failures.ToArray(), NetworkSecuirtyGroupName = nsgname, RuleName = rulename
};
nsgfails.Add(data);
}
}
//Send our Non-Compliant NSGs to a Logic App to handle notification/auditing
var numsent = QueueNSGFailures(nsgfails, log).Result;
//Successfully sent our non-compliant list to logic app update the last run date
var x = GetOrUpdateLastRunStateInfo(currentstate).Result;
log.Info((numsent < 1) ? "No NSG Rules found out of compliance" : "Found and sent " + numsent + " NSG rules out of compliance or deleted to action queue");
log.Info("Last Run Time Updated to: " + currentstate.LastRunTimeUTC);
}
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogWarning("AutoScaleManager:::: Task to check Scaling requirement.. Started (ASM Version : V3.1)");
var subscriptionId = System.Environment.GetEnvironmentVariable("SUBSCRIPTION_ID", EnvironmentVariableTarget.Process);
var resoureGroupName = System.Environment.GetEnvironmentVariable("RESOURCE_GROUP_NAME", EnvironmentVariableTarget.Process);
var vmScalesetName = System.Environment.GetEnvironmentVariable("VMSS_NAME", EnvironmentVariableTarget.Process);
var minFTDCountStr = System.Environment.GetEnvironmentVariable("MIN_FTD_COUNT", EnvironmentVariableTarget.Process);
var maxFTDCountStr = System.Environment.GetEnvironmentVariable("MAX_FTD_COUNT", EnvironmentVariableTarget.Process);
var sampleTimeMin = System.Environment.GetEnvironmentVariable("SAMPLING_TIME_MIN", EnvironmentVariableTarget.Process);
var scaleOutThresholdCpuStr = System.Environment.GetEnvironmentVariable("SCALE_OUT_THRESHLD_CPU", EnvironmentVariableTarget.Process);
var scaleInThresholdCpuStr = System.Environment.GetEnvironmentVariable("SCALE_IN_THRESHLD_CPU", EnvironmentVariableTarget.Process);
var scaleOutThresholdMemStr = System.Environment.GetEnvironmentVariable("SCALE_OUT_THRESHLD_MEM", EnvironmentVariableTarget.Process);
var scaleInThresholdMemStr = System.Environment.GetEnvironmentVariable("SCALE_IN_THRESHLD_MEM", EnvironmentVariableTarget.Process);
var initialDeployMethod = System.Environment.GetEnvironmentVariable("INITIAL_DEPLOYMENT_MODE", EnvironmentVariableTarget.Process); //supported STEP / BULK
var scalingPolicy = System.Environment.GetEnvironmentVariable("SCALING_POLICY", EnvironmentVariableTarget.Process); // POLICY-1 / POLICY-2
var metrics = System.Environment.GetEnvironmentVariable("SCALING_METRICS_LIST", EnvironmentVariableTarget.Process).ToLower();
int minFTDCount = Convert.ToInt32(minFTDCountStr);
int maxFTDCount = Convert.ToInt32(maxFTDCountStr);
double scaleOutThresholdCpu = Convert.ToDouble(scaleOutThresholdCpuStr);
double scaleInThresholdCpu = Convert.ToDouble(scaleInThresholdCpuStr);
double scaleOutThresholdMem = Convert.ToDouble(scaleOutThresholdMemStr);
double scaleInThresholdMem = Convert.ToDouble(scaleInThresholdMemStr);
int currentVmCapacity = 0;
string scaleStr = "";
log.LogInformation("CPU Scale Out threshold: {0}%, Scale In threshold : {1}%", scaleOutThresholdCpu, scaleInThresholdCpu);
// log.LogInformation("Memory Scale Out threshold: {0}%, Scale In threshold : {1}%", scaleOutThresholdMem, scaleInThresholdMem);
//Reject if CPU scale Out Threshold < scale In Threshold
if (scaleOutThresholdCpu <= scaleInThresholdCpu)
{
log.LogError("AutoScaleManager:::: CPU metrics ScaleOut Threshold ({0}) is less than or equal to ScaleIn Threshold ({1}) this is not correct", scaleOutThresholdCpu, scaleInThresholdCpu);
return((ActionResult) new BadRequestObjectResult("ERROR: CPU Metrics ScaleOut threshold is less than or equal to ScaleIn threshold"));
}
//Validate Metrics
if ((!metrics.Contains("cpu")) && (!metrics.Contains("memory")))
{
log.LogError("AutoScaleManager:::: Invalid metrics specified : {0} (valid metrics are CPU or CPU, Memory)", metrics);
return((ActionResult) new BadRequestObjectResult("ERROR: Invalid Metrics..Can not continue"));
}
//Check FMC connection, If we can not connect to FMC do not continue
log.LogInformation("AutoScaleManager:::: Checking FMC connection");
var getAuth = new fmcAuthClass();
string authToken = getAuth.getFmcAuthToken(log);
if ("ERROR" == authToken)
{
log.LogError("AutoScaleManager:::: Failed to connect to FMC..Can not continue");
return((ActionResult) new BadRequestObjectResult("ERROR: Failed to connet to FMC..Can not continue"));
}
log.LogInformation("AutoScaleManager:::: Sampling Resource Utilization at {0}min Average", sampleTimeMin);
var factory = new AzureCredentialsFactory();
var msiCred = factory.FromMSI(new MSILoginInformation(MSIResourceType.AppService), AzureEnvironment.AzureGlobalCloud);
var azure = Azure.Configure().WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic).Authenticate(msiCred).WithSubscription(subscriptionId);
string resourceUri = null;
var vmss = azure.VirtualMachineScaleSets.GetByResourceGroup(resoureGroupName, vmScalesetName);
resourceUri = vmss.Id;
if (null == resourceUri)
{
log.LogError("AutoScaleManager:::: Unable to get resource uri");
return((ActionResult) new BadRequestObjectResult("ERROR: Unable to get resource uri"));
}
currentVmCapacity = vmss.Capacity;
log.LogWarning("AutoScaleManager:::: Current capacity of VMSS : {0}", currentVmCapacity);
//If the VMSS capacity is '0' consider this as first deployment and spawn 'minimum FTD count' at a time
if ((0 == currentVmCapacity) && (0 != minFTDCount))
{
log.LogWarning("AutoScaleManager:::: Current VMSS capacity is 0, considering it as first deployment (min FTD count needed : {0}", minFTDCount);
if ("BULK" == initialDeployMethod)
{
log.LogWarning("AutoScaleManager:::: Selected initial deployment mode is BULK");
log.LogWarning("AutoScaleManager:::: Deploying {0} number of FTDvs in scale set", minFTDCount);
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"" + minFTDCount + "\", \"TYPE\": \"INIT\" }";
return((ActionResult) new OkObjectResult(scaleStr));
}
else
{
log.LogWarning("AutoScaleManager:::: BULK method is not selected for initial deployment.. proceeding with STEP");
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"1\", \"TYPE\": \"REGULAR\"}";
return((ActionResult) new OkObjectResult(scaleStr));
}
}
//If current capacity is less than minimum FTD count requied then we need to scale-out
if (currentVmCapacity < minFTDCount)
{
log.LogWarning("AutoScaleManager:::: Current VMSS Capacity({0}) is less than minimum FTD count ({1}) needed.. time to SCALE-OUT", currentVmCapacity, minFTDCount);
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"1\", \"TYPE\": \"REGULAR\"}";
return((ActionResult) new OkObjectResult(scaleStr));
}
//-------------------------------------------------Scaling decission based on Metrics------------------------------------------------------
log.LogWarning("AutoScaleManager:::: Scaling Policy : {0}", scalingPolicy);
var sampleIntervalMin = System.TimeSpan.FromMinutes(Convert.ToDouble(sampleTimeMin));
MonitorManagementClient metricClient = new MonitorManagementClient(msiCred);
double ftdCpuUsage = 0;
double groupCpuUsage = 0;
double consolidatedCpuUsage = 0;
bool scaleInRejectFlag = false;
double minFtdCpuUsage = 9999;
string leastCpuLoadedFtd = "";
string leastCpuLoadedFtdIndex = "";
bool memoryMetricsEnabled = false;
double ftdMemUsage = 0;
double groupMemUsage = 0;
double consolidatedMemUsage = 0;
string ftdNameWithHighMemUtilization = "";
//Get FTD's Memory if 'Memory' metrics is enabled
if (metrics.Contains("memory"))
{
memoryMetricsEnabled = true;
log.LogInformation("Memory metrics enabled");
log.LogInformation("Memory Scale Out threshold: {0}%, Scale In threshold : {1}%", scaleOutThresholdMem, scaleInThresholdMem);
//Reject if Memory scale Out Threshold < scale In Threshold
if (scaleOutThresholdMem <= scaleInThresholdMem)
{
log.LogError("AutoScaleManager:::: Memory metrics ScaleOut Threshold ({0}) is less than or equal to ScaleIn Threshold ({1}) this is not correct", scaleOutThresholdMem, scaleInThresholdMem);
return((ActionResult) new BadRequestObjectResult("ERROR: Memory Metrics ScaleOut threshold is less than or equal to ScaleIn threshold"));
}
var getMetrics = new getMetricsClass();
var getId = new getDevIdByNameClass();
var devIds = getId.getAllDevId(authToken, log);
if ("ERROR" == devIds)
{
log.LogError("AutoScaleManager::::Unable to get device IDs");
return((ActionResult) new StatusCodeResult(StatusCodes.Status500InternalServerError));
}
//parse json object
JObject o = JObject.Parse(devIds);
foreach (var vm in vmss.VirtualMachines.List())
{
var vmName = vm.Name.ToString();
var devId = "";
try
{
foreach (var item in o["items"])
{
if (vmName == item["name"].ToString())
{
devId = item["id"].ToString();
break;
}
}
if (0 == devId.Length)
{
log.LogError("AutoScaleManager:::: Unable to get Device ID for Device Name({0})", vmName);
return((ActionResult) new StatusCodeResult(StatusCodes.Status500InternalServerError));
}
}
catch
{
log.LogError("AutoScaleManager:::: Exception Occoured while parsing device id response");
return((ActionResult) new StatusCodeResult(StatusCodes.Status500InternalServerError));
}
ftdMemUsage = Convert.ToDouble(getMetrics.getFtdMemoryMetrics(devId, authToken, log));
if (-1 == ftdMemUsage)
{
log.LogError("AutoScaleManager:::: Failed to get Memory usage of {0}", vmName);
return((ActionResult) new StatusCodeResult(StatusCodes.Status500InternalServerError));
}
if (ftdMemUsage > scaleInThresholdMem)
{
//No need to Scale-In
scaleInRejectFlag = true;
}
log.LogInformation("AutoScaleManager:::: Memory usage of {0} is {1} %", vmName, ftdMemUsage);
if ("POLICY-1" == scalingPolicy)
{
if (ftdMemUsage > scaleOutThresholdMem)
{
log.LogWarning("AutoScaleManager:::: FTD {0} has Memory Utilization of {1} % which is greater than Scale Out threshold", vmName, ftdMemUsage);
ftdNameWithHighMemUtilization = vmName;
break;
}
}
else if ("POLICY-2" == scalingPolicy)
{
groupMemUsage += ftdMemUsage;
}
}
groupMemUsage /= vmss.Capacity;
if ("POLICY-2" == scalingPolicy)
{
log.LogInformation("AutoScaleManager:::: Group Memory average usage : {0} %", groupMemUsage);
}
}
else
{
//Memory metrics not enabled, reset thresholds
scaleOutThresholdMem = 0;
}
if ("POLICY-2" == scalingPolicy)
{
log.LogInformation("AutoScaleManager:::: Scaling Policy-2 Selected..Getting average CPU utilization of scale set");
var response = await metricClient.Metrics.ListAsync(resourceUri, null, null, sampleIntervalMin, "Percentage CPU", "Average");
foreach (var metric in response.Value)
{
foreach (var series in metric.Timeseries)
{
foreach (var point in series.Data)
{
if (point.Average.HasValue)
{
groupCpuUsage = point.Average.Value;
log.LogDebug("AutoScaleManager:::: avg cpu: {0}", groupCpuUsage);
}
}
}
}
log.LogInformation("AutoScaleManager:::: Group CPU average usage : {0} %", groupCpuUsage);
}
foreach (var vm in vmss.VirtualMachines.List())
{
var vmName = vm.Name;
ftdCpuUsage = 0;
//Metrics filter
ODataQuery <MetadataValue> odataFilterMetrics = new ODataQuery <MetadataValue>(string.Format("VMName eq '{0}'", vmName));
// log.LogInformation("AutoScaleManager:::: Getting Metrics for : {0}", vmName);
var response = await metricClient.Metrics.ListAsync(resourceUri, odataFilterMetrics, null, sampleIntervalMin, "Percentage CPU", "Average");
foreach (var metric in response.Value)
{
foreach (var series in metric.Timeseries)
{
foreach (var point in series.Data)
{
if (point.Average.HasValue)
{
ftdCpuUsage = point.Average.Value;
log.LogDebug("AutoScaleManager:::: avg cpu: {0}", ftdCpuUsage);
}
}
}
}
log.LogInformation("AutoScaleManager:::: Avg CPU Utilizatio of VM({0}) in last {1}min : {2}%", vmName, sampleTimeMin, ftdCpuUsage);
//Maintain the FTD with minimum utilization to scale-in if needed
if (ftdCpuUsage < minFtdCpuUsage)
{
minFtdCpuUsage = ftdCpuUsage;
leastCpuLoadedFtd = vmName;
leastCpuLoadedFtdIndex = vm.InstanceId;
}
if ("POLICY-1" == scalingPolicy)
{
//Average usage of individual Instance
consolidatedCpuUsage = ftdCpuUsage;
consolidatedMemUsage = ftdMemUsage;
}
else if ("POLICY-2" == scalingPolicy)
{
//Scale Set average utilization
consolidatedCpuUsage = groupCpuUsage;
consolidatedMemUsage = groupMemUsage;
}
else
{
log.LogError("AutoScaleManager:::: Invalid Scaling Policy {0}", scalingPolicy);
return((ActionResult) new BadRequestObjectResult("ERROR: Invalid Scaling Policy"));
}
//If CPU utilization is greater than scale-out threshold then Scale-Out
//Note: if memory metrics is not enabled then consolidatedMemUsage will be always 0
if ((consolidatedCpuUsage > scaleOutThresholdCpu) || (consolidatedMemUsage > scaleOutThresholdMem))
{
//If current capacity is equal to max FTD count required then do nothing
//If current capacity is more than max FTD count (This should never happen) do nothing
if (currentVmCapacity >= maxFTDCount)
{
log.LogWarning("AutoScaleManager:::: Current VMSS Capacity({0}) is greater than or equal to max FTD count ({1}) needed.. No action needed", currentVmCapacity, maxFTDCount);
return((ActionResult) new OkObjectResult("NOACTION"));
}
if ("POLICY-1" == scalingPolicy)
{
log.LogWarning("AutoScaleManager:::: Avg CPU Utilizatio of VM({0}) in last {1}min is {2}% ", vmName, sampleTimeMin, consolidatedCpuUsage);
if (memoryMetricsEnabled && (consolidatedMemUsage > scaleOutThresholdMem))
{
log.LogWarning("AutoScaleManager:::: Avg Memory Utilizatio of VM({0}) is {1}% ", ftdNameWithHighMemUtilization, consolidatedMemUsage);
}
log.LogWarning("AutoScaleManager:::: Time to SCALE OUT");
}
else if ("POLICY-2" == scalingPolicy)
{
log.LogWarning("AutoScaleManager:::: Avg CPU Utilizatio of Scale Set in last {0}min is {1}% ", sampleTimeMin, consolidatedCpuUsage);
if (memoryMetricsEnabled)
{
log.LogWarning("AutoScaleManager:::: Avg Memory Utilizatio of Scale Set is {0}% ", consolidatedMemUsage);
}
log.LogWarning("AutoScaleManager:::: Average resource utilization of scale set is more than Scale Out threshold.. Time to SCALE OUT");
}
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"1\", \"TYPE\": \"REGULAR\" }";
return((ActionResult) new OkObjectResult(scaleStr));
}
//If any VM's CPU utilization is greater than scale-in threshold then Scale-In is not needed
else if (ftdCpuUsage > scaleInThresholdCpu)
{
scaleInRejectFlag = true;
}
}
//if scaleInRejectFlag is not set, it means all the VM's CPU & Memory utilization is less than or equal to Scale-In threshold
//Hence considering only least CPU consuming FTDv for Scale-In operation
if (false == scaleInRejectFlag)
{
//If current capacity is less than or equal to minimum FTD count requied then scale-in should not be done
if (currentVmCapacity <= minFTDCount)
{
log.LogWarning("AutoScaleManager:::: Scale-In needed but Current VMSS Capacity({0}) is less than or equal to minimum FTD count ({1}) needed.. No Action done", currentVmCapacity, minFTDCount);
return((ActionResult) new OkObjectResult("NOACTION"));
}
var networkInterfaceName = System.Environment.GetEnvironmentVariable("MNGT_NET_INTERFACE_NAME", EnvironmentVariableTarget.Process);
var ipConfigurationName = System.Environment.GetEnvironmentVariable("MNGT_IP_CONFIG_NAME", EnvironmentVariableTarget.Process);
var publicIpAddressName = System.Environment.GetEnvironmentVariable("MNGT_PUBLIC_IP_NAME", EnvironmentVariableTarget.Process);
var NmClient = new NetworkManagementClient(msiCred)
{
SubscriptionId = azure.SubscriptionId
};
var publicIp = NmClient.PublicIPAddresses.GetVirtualMachineScaleSetPublicIPAddress(resoureGroupName, vmScalesetName, leastCpuLoadedFtdIndex, networkInterfaceName, ipConfigurationName, publicIpAddressName).IpAddress;
log.LogWarning("AutoScaleManager:::: CPU Utilization of all the FTD's is less than or equal to CPU Scale-In threshold({0}%).. Time to SCALE-IN", scaleInThresholdCpu);
if (memoryMetricsEnabled)
{
log.LogWarning("AutoScaleManager:::: Memory Utilization of all the FTD's is less than or equal to Memory Scale-In threshold({0}%).. Time to SCALE-IN", scaleInThresholdMem);
}
log.LogWarning("AutoScaleManager:::: Least loaded FTD is : {0} with Utilization : {1}%", leastCpuLoadedFtd, minFtdCpuUsage);
scaleStr = "{ \"COMMAND\": \"SCALEIN\", \"ftdDevName\": \"" + leastCpuLoadedFtd + "\", \"ftdPublicIp\": \"" + publicIp + "\", \"instanceid\" : \"" + leastCpuLoadedFtdIndex + "\" }";
return((ActionResult) new OkObjectResult(scaleStr));
}
//Scaling not needed
log.LogWarning("AutoScaleManager:::: FTD scaleset utilization is within threshold.. no action needed");
return((ActionResult) new OkObjectResult("NOACTION"));
}
private void DeleteMultiResSubscriptionLevelDynamicMetricAlertRule(MonitorManagementClient insightsClient)
{
insightsClient.MetricAlerts.Delete(resourceGroupName: ResourceGroupName, ruleName: MultiResSubscriptionLevelDynamicRuleName);
}
private void DeleteMetricAlertRule(MonitorManagementClient insightsClient)
{
insightsClient.MetricAlerts.Delete(resourceGroupName: ResourceGroupName, ruleName: RuleName);
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogWarning("AutoScaleManager:::: Task to check Scaling requirement.. Started (ASM Version : V2.0)");
var resoureGroupName = System.Environment.GetEnvironmentVariable("RESOURCE_GROUP_NAME", EnvironmentVariableTarget.Process);
var vmScalesetName = System.Environment.GetEnvironmentVariable("VMSS_NAME", EnvironmentVariableTarget.Process);
var minFTDCountStr = System.Environment.GetEnvironmentVariable("MIN_FTD_COUNT", EnvironmentVariableTarget.Process);
var maxFTDCountStr = System.Environment.GetEnvironmentVariable("MAX_FTD_COUNT", EnvironmentVariableTarget.Process);
var sampleTimeMin = System.Environment.GetEnvironmentVariable("SAMPLING_TIME_MIN", EnvironmentVariableTarget.Process);
var scaleOutThresholdStr = System.Environment.GetEnvironmentVariable("SCALE_OUT_THRESHLD", EnvironmentVariableTarget.Process);
var scaleInThresholdStr = System.Environment.GetEnvironmentVariable("SCALE_IN_THRESHLD", EnvironmentVariableTarget.Process);
var initialDeployMethod = System.Environment.GetEnvironmentVariable("INITIAL_DEPLOYMENT_MODE", EnvironmentVariableTarget.Process); //supported STEP / BULK
var scalingPolicy = System.Environment.GetEnvironmentVariable("SCALING_POLICY", EnvironmentVariableTarget.Process); // POLICY-1 / POLICY-2
var subscriptionId = System.Environment.GetEnvironmentVariable("SUBSCRIPTION_ID", EnvironmentVariableTarget.Process);
int minFTDCount = Convert.ToInt32(minFTDCountStr);
int maxFTDCount = Convert.ToInt32(maxFTDCountStr);
double scaleOutThreshold = Convert.ToDouble(scaleOutThresholdStr);
double scaleInThreshold = Convert.ToDouble(scaleInThresholdStr);
int currentVmCapacity = 0;
string scaleStr = "";
//Reject if scaleOutThreshold < scaleInThreshold
if (scaleOutThreshold <= scaleInThreshold)
{
log.LogError("AutoScaleManager:::: ScaleOut Threshold ({0}) is less than or equal to ScaleIn Threshold ({1}) this is not correct", scaleOutThreshold, scaleInThreshold);
return((ActionResult) new BadRequestObjectResult("ERROR: ScaleOut threshold is less than or equal to ScaleIn threshold"));
}
//Check FMC connection, If we can not connect to FMC do not continue
log.LogInformation("AutoScaleManager:::: Checking FMC connection");
var getAuth = new fmcAuthClass();
string authToken = getAuth.getFmcAuthToken(log);
if ("ERROR" == authToken)
{
log.LogError("AutoScaleManager:::: Failed to connect to FMC..Can not continue");
return((ActionResult) new BadRequestObjectResult("ERROR: Failed to connet to FMC..Can not continue"));
}
log.LogInformation("AutoScaleManager:::: Sampling Resource Utilization at {0}min Average", sampleTimeMin);
var factory = new AzureCredentialsFactory();
var msiCred = factory.FromMSI(new MSILoginInformation(MSIResourceType.AppService), AzureEnvironment.AzureGlobalCloud);
var azure = Azure.Configure().WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic).Authenticate(msiCred).WithSubscription(subscriptionId);
string resourceUri = null;
var vmss = azure.VirtualMachineScaleSets.GetByResourceGroup(resoureGroupName, vmScalesetName);
resourceUri = vmss.Id;
if (null == resourceUri)
{
log.LogError("AutoScaleManager:::: Unable to get resource uri");
return((ActionResult) new BadRequestObjectResult("ERROR: Unable to get resource uri"));
}
currentVmCapacity = vmss.Capacity;
log.LogWarning("AutoScaleManager:::: Current capacity of VMSS : {0}", currentVmCapacity);
//If the VMSS capacity is '0' consider this as first deployment and spawn 'minimum FTD count' at a time
if ((0 == currentVmCapacity) && (0 != minFTDCount))
{
log.LogWarning("AutoScaleManager:::: Current VMSS capacity is 0, considering it as first deployment (min FTD count needed : {0}", minFTDCount);
if ("BULK" == initialDeployMethod)
{
log.LogWarning("AutoScaleManager:::: Selected initial deployment mode is BULK");
log.LogWarning("AutoScaleManager:::: Deploying {0} number of FTDvs in scale set", minFTDCount);
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"" + minFTDCount + "\", \"TYPE\": \"INIT\" }";
return((ActionResult) new OkObjectResult(scaleStr));
}
else
{
log.LogWarning("AutoScaleManager:::: BULK method is not selected for initial deployment.. proceeding with STEP");
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"1\", \"TYPE\": \"REGULAR\"}";
return((ActionResult) new OkObjectResult(scaleStr));
}
}
//If current capacity is less than minimum FTD count requied then we need to scale-out
if (currentVmCapacity < minFTDCount)
{
log.LogWarning("AutoScaleManager:::: Current VMSS Capacity({0}) is less than minimum FTD count ({1}) needed.. time to SCALE-OUT", currentVmCapacity, minFTDCount);
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"1\", \"TYPE\": \"REGULAR\"}";
return((ActionResult) new OkObjectResult(scaleStr));
}
//-------------------------------------------------Scaling decission based on Metrics------------------------------------------------------
var sampleIntervalMin = System.TimeSpan.FromMinutes(Convert.ToDouble(sampleTimeMin));
MonitorManagementClient metricClient = new MonitorManagementClient(msiCred);
double ftdUsage = 0;
double groupUsage = 0;
double consolidatedUsage = 0;
bool scaleInRejectFlag = false;
double minFtdUsage = 9999;
string leastLoadedFtd = "";
string leastLoadedFtdIndex = "";
log.LogWarning("AutoScaleManager:::: Scaling Policy : {0}", scalingPolicy);
if ("POLICY-2" == scalingPolicy)
{
log.LogInformation("AutoScaleManager:::: Scaling Policy-2 Selected..Getting average CPU utilization of scale set");
var response = await metricClient.Metrics.ListAsync(resourceUri, null, null, sampleIntervalMin, "Percentage CPU", "Average");
foreach (var metric in response.Value)
{
foreach (var series in metric.Timeseries)
{
foreach (var point in series.Data)
{
if (point.Average.HasValue)
{
groupUsage = point.Average.Value;
log.LogDebug("AutoScaleManager:::: avg cpu: {0}", ftdUsage);
}
}
}
}
log.LogInformation("AutoScaleManager:::: Group average usage : {0}", groupUsage);
}
foreach (var vm in vmss.VirtualMachines.List())
{
var vmName = vm.Name;
ftdUsage = 0;
//Metrics filter
ODataQuery <MetadataValue> odataFilterMetrics = new ODataQuery <MetadataValue>(string.Format("VMName eq '{0}'", vmName));
log.LogInformation("AutoScaleManager:::: Getting Metrics for : {0}", vmName);
var response = await metricClient.Metrics.ListAsync(resourceUri, odataFilterMetrics, null, sampleIntervalMin, "Percentage CPU", "Average");
foreach (var metric in response.Value)
{
foreach (var series in metric.Timeseries)
{
foreach (var point in series.Data)
{
if (point.Average.HasValue)
{
ftdUsage = point.Average.Value;
log.LogDebug("AutoScaleManager:::: avg cpu: {0}", ftdUsage);
}
}
}
}
log.LogInformation("AutoScaleManager:::: Avg CPU Utilizatio of VM({0}) in last {1}min : {2}%", vmName, sampleTimeMin, ftdUsage);
//Maintain the FTD with minimum utilization to scale-in if needed
if (ftdUsage < minFtdUsage)
{
minFtdUsage = ftdUsage;
leastLoadedFtd = vmName;
leastLoadedFtdIndex = vm.InstanceId;
}
if ("POLICY-1" == scalingPolicy)
{
//Average usage of individual Instance
consolidatedUsage = ftdUsage;
}
else if ("POLICY-2" == scalingPolicy)
{
//Scale Set average utilization
consolidatedUsage = groupUsage;
}
else
{
log.LogError("Invalid Scaling Policy {0}", scalingPolicy);
return((ActionResult) new BadRequestObjectResult("ERROR: Invalid Scaling Policy"));
}
//If CPU utilization is greater than scale-out threshold then Scale-Out
if (consolidatedUsage > scaleOutThreshold)
{
//If current capacity is equal to max FTD count required then do nothing
//If current capacity is more than max FTD count (This should never happen) do nothing
if (currentVmCapacity >= maxFTDCount)
{
log.LogWarning("AutoScaleManager:::: Current VMSS Capacity({0}) is greater than or equal to max FTD count ({1}) needed.. No action needed", currentVmCapacity, maxFTDCount);
return((ActionResult) new OkObjectResult("NOACTION"));
}
if ("POLICY-1" == scalingPolicy)
{
log.LogWarning("AutoScaleManager:::: Avg CPU Utilizatio of VM({0}) in last {1}min is {2}% which is greater than ScaleOut threshold({3}%) .. Time to SCALE-OUT", vmName, sampleTimeMin, consolidatedUsage, scaleOutThreshold);
}
else if ("POLICY-2" == scalingPolicy)
{
log.LogWarning("AutoScaleManager:::: Avg CPU Utilizatio of Scale Set in last {0}min is {1}% which is greater than ScaleOut threshold({2}%) .. Time to SCALE-OUT", sampleTimeMin, consolidatedUsage, scaleOutThreshold);
}
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"1\", \"TYPE\": \"REGULAR\" }";
return((ActionResult) new OkObjectResult(scaleStr));
}
//If any VM's CPU utilization is greater than scale-in threshold then Scale-In is not needed
else if (ftdUsage > scaleInThreshold)
{
scaleInRejectFlag = true;
}
}
//if scaleInRejectFlag is not set, it means all the VM's CPU utilization is less than or equal to Scale-In threshold
if (false == scaleInRejectFlag)
{
//If current capacity is less than or equal to minimum FTD count requied then scale-in should not be done
if (currentVmCapacity <= minFTDCount)
{
log.LogWarning("AutoScaleManager:::: Scale-In needed but Current VMSS Capacity({0}) is less than or equal to minimum FTD count ({1}) needed.. No Action done", currentVmCapacity, minFTDCount);
return((ActionResult) new OkObjectResult("NOACTION"));
}
var networkInterfaceName = System.Environment.GetEnvironmentVariable("MNGT_NET_INTERFACE_NAME", EnvironmentVariableTarget.Process);
var ipConfigurationName = System.Environment.GetEnvironmentVariable("MNGT_IP_CONFIG_NAME", EnvironmentVariableTarget.Process);
var publicIpAddressName = System.Environment.GetEnvironmentVariable("MNGT_PUBLIC_IP_NAME", EnvironmentVariableTarget.Process);
var NmClient = new NetworkManagementClient(msiCred)
{
SubscriptionId = azure.SubscriptionId
};
var publicIp = NmClient.PublicIPAddresses.GetVirtualMachineScaleSetPublicIPAddress(resoureGroupName, vmScalesetName, leastLoadedFtdIndex, networkInterfaceName, ipConfigurationName, publicIpAddressName).IpAddress;
log.LogWarning("AutoScaleManager:::: CPU Utilization of all the FTD's is less than or equal to Scale-In threshold({0}%).. Time to SCALE-IN", scaleInThreshold);
log.LogWarning("AutoScaleManager:::: Least loaded FTD is : {0} with Utilization : {1}%", leastLoadedFtd, minFtdUsage);
scaleStr = "{ \"COMMAND\": \"SCALEIN\", \"ftdDevName\": \"" + leastLoadedFtd + "\", \"ftdPublicIp\": \"" + publicIp + "\", \"instanceid\" : \"" + leastLoadedFtdIndex + "\" }";
return((ActionResult) new OkObjectResult(scaleStr));
}
//Scaling not needed
log.LogWarning("AutoScaleManager:::: FTD scaleset utilization is within threshold.. no action needed");
return((ActionResult) new OkObjectResult("NOACTION"));
}
