async Task <BastionHost> GetResourceInternalAsync(string resourceGroupName, string bastionHostName, bool failIfNotFound = true)
{
try
{
using (var client = new Microsoft.Azure.Management.Network.NetworkManagementClient(_credentials))
{
client.SubscriptionId = _subscriptionId;
var bastion = await client.BastionHosts.GetAsync(resourceGroupName, bastionHostName);
return(bastion);
}
}
catch (Exception ex)
{
if (ex.Message.ToLower().Contains("could not be found") || ex.Message.ToLower().Contains("was not found"))
{
if (failIfNotFound)
{
throw NotFoundException.CreateForAzureResource(bastionHostName, resourceGroupName, ex);
}
else
{
return(null);
}
}
throw;
}
}
async Task DeleteInternal(string resourceGroupName, string bastionHostName)
{
var bastion = await GetResourceInternalAsync(resourceGroupName, bastionHostName, false);
if (bastion == null)
{
//Allready deleted
_logger.LogWarning($"Deleting resource {bastionHostName} failed because it was not found. Assuming allready deleted");
return;
}
using (var client = new Microsoft.Azure.Management.Network.NetworkManagementClient(_credentials))
{
client.SubscriptionId = _subscriptionId;
CheckIfResourceHasCorrectManagedByTagThrowIfNot(resourceGroupName, bastion.Tags);
await client.BastionHosts.DeleteAsync(resourceGroupName, bastionHostName);
}
}
async Task <BastionHost> CreateInternal(Region region, string resourceGroupName, string bastionName, string subnetId, Dictionary <string, string> tags, CancellationToken cancellationToken = default)
{
var publicIpName = AzureResourceNameUtil.BastionPublicIp(bastionName);
var pip = await _azure.PublicIPAddresses.Define(publicIpName)
.WithRegion(region)
.WithExistingResourceGroup(resourceGroupName)
.WithStaticIP()
.WithSku(PublicIPSkuType.Standard)
.WithTags(tags)
.CreateAsync(cancellationToken);
using (var client = new Microsoft.Azure.Management.Network.NetworkManagementClient(_credentials))
{
client.SubscriptionId = _subscriptionId;
var ipConfigs = new List <BastionHostIPConfiguration> {
new BastionHostIPConfiguration()
{
Name = $"{bastionName}-ip-config",
Subnet = new SubResource(subnetId),
PrivateIPAllocationMethod = "Dynamic",
PublicIPAddress = new SubResource(pip.Inner.Id),
}
};
var bastion = new BastionHost()
{
Location = region.Name,
IpConfigurations = ipConfigs,
Tags = tags
};
var createdBastion = await client.BastionHosts.CreateOrUpdateAsync(resourceGroupName, bastionName, bastion, cancellationToken);
return(createdBastion);
}
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
var subscriptionId = System.Environment.GetEnvironmentVariable("SUBSCRIPTION_ID", EnvironmentVariableTarget.Process);
string COUNT = req.Query["COUNT"];
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic data = JsonConvert.DeserializeObject(requestBody);
COUNT = COUNT ?? data?.COUNT;
string TYPE = req.Query["TYPE"];
string requestBody1 = await new StreamReader(req.Body).ReadToEndAsync();
dynamic data1 = JsonConvert.DeserializeObject(requestBody);
TYPE = TYPE ?? data?.TYPE;
int ftdCountInt = Convert.ToInt32(COUNT);
int index = 1;
if ("REGULAR" == TYPE)
{
log.LogWarning("GetFtdPublicIp:::: This is regular scale-out ");
}
else if ("INIT" == TYPE)
{
log.LogWarning("GetFtdPublicIp:::: This is initial deployment");
}
else
{
return((ActionResult) new BadRequestObjectResult("ERROR: Invalid request TYPE"));
}
var resoureGroupName = System.Environment.GetEnvironmentVariable("RESOURCE_GROUP_NAME", EnvironmentVariableTarget.Process);
var vmScalesetName = System.Environment.GetEnvironmentVariable("VMSS_NAME", EnvironmentVariableTarget.Process);
var networkInterfaceName = System.Environment.GetEnvironmentVariable("MNGT_NET_INTERFACE_NAME", EnvironmentVariableTarget.Process);
var ipConfigurationName = System.Environment.GetEnvironmentVariable("MNGT_IP_CONFIG_NAME", EnvironmentVariableTarget.Process);
var publicIpAddressName = System.Environment.GetEnvironmentVariable("MNGT_PUBLIC_IP_NAME", EnvironmentVariableTarget.Process);
log.LogWarning("GetFtdPublicIp:::: Getting Public IP of new FTD (RG : {0}, VMSS: {1} )", resoureGroupName.ToString(), vmScalesetName.ToString());
log.LogInformation("GetFtdPublicIp:::: Network Interface name : {0}, IP Configuration Name : {1}, Public IP Address Name : {2}", networkInterfaceName, ipConfigurationName, publicIpAddressName);
var factory = new AzureCredentialsFactory();
var msiCred = factory.FromMSI(new MSILoginInformation(MSIResourceType.AppService), AzureEnvironment.AzureGlobalCloud);
var azure = Azure.Configure().WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic).Authenticate(msiCred).WithSubscription(subscriptionId);
var NmClient = new NetworkManagementClient(msiCred)
{
SubscriptionId = azure.SubscriptionId
};
var interfaceList = NmClient.NetworkInterfaces.ListVirtualMachineScaleSetNetworkInterfaces(resoureGroupName, vmScalesetName);
string vmindex = "";
string tmpVmindex = "";
int intVmindex = 0;
var vmlist = azure.VirtualMachineScaleSets.GetByResourceGroup(resoureGroupName, vmScalesetName);
var vmStatus = "";
var tmpVmName = "ERROR";
//ToDo: This logic should be simplified with just one loop of vmlist, no need of interfaceList
foreach (var netInterface in interfaceList)
{
if (netInterface.IpConfigurations[0].PublicIPAddress != null)
{
var tmpIntfName = netInterface.IpConfigurations[0].PublicIPAddress.Id.Split('/').GetValue(12);
var tmpConfigName = netInterface.IpConfigurations[0].PublicIPAddress.Id.Split('/').GetValue(14);
var tmpPubIpName = netInterface.IpConfigurations[0].PublicIPAddress.Id.Split('/').GetValue(16);
if ((tmpIntfName.ToString() == networkInterfaceName) && (tmpConfigName.ToString() == ipConfigurationName) && (tmpPubIpName.ToString() == publicIpAddressName))
{
vmindex = netInterface.IpConfigurations[0].PublicIPAddress.Id.Split('/').GetValue(10).ToString();
vmStatus = "ON";
foreach (var vm in vmlist.VirtualMachines.List())
{
if (vm.InstanceId == vmindex)
{
if (null == vm.PowerState)
{
vmStatus = "OFF";
}
if (null != vm.Name)
{
tmpVmName = vm.Name;
}
break;
}
}
//Azure bug, VM will be present in Azure DB for long time even after deletion
if ("OFF" == vmStatus)
{
log.LogError("GetFtdPublicIp:::: VM index :{0} is in unknown state..skip", vmindex);
continue;
}
//Azure bug, some times even deleted VMs are still attahed to network interfaces
if ("ERROR" == tmpVmName)
{
log.LogError("GetFtdPublicIp:::: VM index :{0} VM name not found...skip", vmindex);
continue;
}
if ("INIT" == TYPE)
{
if (index == ftdCountInt)
{
//index >100 is just to safegaurd this loop..its has no other logic
break;
}
index++;
}
else
{
//Azure bug: Some time it will mix indexes and does not preserve sequence
if (Convert.ToInt32(vmindex) < intVmindex)
{
log.LogWarning("GetFtdPublicIp:::: Azure index jumbling detected");
vmindex = intVmindex.ToString();
}
else
{
intVmindex = Convert.ToInt32(vmindex);
log.LogInformation("GetFtdPublicIp:::: Assigning vmindex = {0}", vmindex);
}
}
}
}
}
var publicIp = NmClient.PublicIPAddresses.GetVirtualMachineScaleSetPublicIPAddress(resoureGroupName, vmScalesetName, vmindex, networkInterfaceName, ipConfigurationName, publicIpAddressName).IpAddress;
if (null == publicIp)
{
log.LogError("GetFtdPublicIp:::: Unable to get Public IP of new FTD (index {0}", vmindex);
return((ActionResult) new BadRequestObjectResult("ERROR: Unable to get Public IP of new FTD"));
}
log.LogInformation("GetFtdPublicIp:::: Public IP of New FTD (VM index {0}) = {1}", vmindex, publicIp);
//find VM name from index
string vmname = "";
string privateIp = "";
var vmss = azure.VirtualMachineScaleSets.GetByResourceGroup(resoureGroupName, vmScalesetName);
foreach (var vm in vmss.VirtualMachines.List())
{
if (vm.InstanceId == vmindex)
{
vmname = vm.Name;
foreach (var netintf in vm.ListNetworkInterfaces())
{
privateIp = netintf.PrimaryPrivateIP;
break;
}
break;
}
}
var commandStr = "{ \"ftdDevName\": \"" + vmname + "\", \"ftdPublicIp\": \"" + publicIp + "\", \"ftdPrivateIp\" : \"" + privateIp + "\" }";
return((ActionResult) new OkObjectResult(commandStr));
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogWarning("AutoScaleManager:::: Task to check Scaling requirement.. Started (ASM Version : V3.1)");
var subscriptionId = System.Environment.GetEnvironmentVariable("SUBSCRIPTION_ID", EnvironmentVariableTarget.Process);
var resoureGroupName = System.Environment.GetEnvironmentVariable("RESOURCE_GROUP_NAME", EnvironmentVariableTarget.Process);
var vmScalesetName = System.Environment.GetEnvironmentVariable("VMSS_NAME", EnvironmentVariableTarget.Process);
var minFTDCountStr = System.Environment.GetEnvironmentVariable("MIN_FTD_COUNT", EnvironmentVariableTarget.Process);
var maxFTDCountStr = System.Environment.GetEnvironmentVariable("MAX_FTD_COUNT", EnvironmentVariableTarget.Process);
var sampleTimeMin = System.Environment.GetEnvironmentVariable("SAMPLING_TIME_MIN", EnvironmentVariableTarget.Process);
var scaleOutThresholdCpuStr = System.Environment.GetEnvironmentVariable("SCALE_OUT_THRESHLD_CPU", EnvironmentVariableTarget.Process);
var scaleInThresholdCpuStr = System.Environment.GetEnvironmentVariable("SCALE_IN_THRESHLD_CPU", EnvironmentVariableTarget.Process);
var scaleOutThresholdMemStr = System.Environment.GetEnvironmentVariable("SCALE_OUT_THRESHLD_MEM", EnvironmentVariableTarget.Process);
var scaleInThresholdMemStr = System.Environment.GetEnvironmentVariable("SCALE_IN_THRESHLD_MEM", EnvironmentVariableTarget.Process);
var initialDeployMethod = System.Environment.GetEnvironmentVariable("INITIAL_DEPLOYMENT_MODE", EnvironmentVariableTarget.Process); //supported STEP / BULK
var scalingPolicy = System.Environment.GetEnvironmentVariable("SCALING_POLICY", EnvironmentVariableTarget.Process); // POLICY-1 / POLICY-2
var metrics = System.Environment.GetEnvironmentVariable("SCALING_METRICS_LIST", EnvironmentVariableTarget.Process).ToLower();
int minFTDCount = Convert.ToInt32(minFTDCountStr);
int maxFTDCount = Convert.ToInt32(maxFTDCountStr);
double scaleOutThresholdCpu = Convert.ToDouble(scaleOutThresholdCpuStr);
double scaleInThresholdCpu = Convert.ToDouble(scaleInThresholdCpuStr);
double scaleOutThresholdMem = Convert.ToDouble(scaleOutThresholdMemStr);
double scaleInThresholdMem = Convert.ToDouble(scaleInThresholdMemStr);
int currentVmCapacity = 0;
string scaleStr = "";
log.LogInformation("CPU Scale Out threshold: {0}%, Scale In threshold : {1}%", scaleOutThresholdCpu, scaleInThresholdCpu);
// log.LogInformation("Memory Scale Out threshold: {0}%, Scale In threshold : {1}%", scaleOutThresholdMem, scaleInThresholdMem);
//Reject if CPU scale Out Threshold < scale In Threshold
if (scaleOutThresholdCpu <= scaleInThresholdCpu)
{
log.LogError("AutoScaleManager:::: CPU metrics ScaleOut Threshold ({0}) is less than or equal to ScaleIn Threshold ({1}) this is not correct", scaleOutThresholdCpu, scaleInThresholdCpu);
return((ActionResult) new BadRequestObjectResult("ERROR: CPU Metrics ScaleOut threshold is less than or equal to ScaleIn threshold"));
}
//Validate Metrics
if ((!metrics.Contains("cpu")) && (!metrics.Contains("memory")))
{
log.LogError("AutoScaleManager:::: Invalid metrics specified : {0} (valid metrics are CPU or CPU, Memory)", metrics);
return((ActionResult) new BadRequestObjectResult("ERROR: Invalid Metrics..Can not continue"));
}
//Check FMC connection, If we can not connect to FMC do not continue
log.LogInformation("AutoScaleManager:::: Checking FMC connection");
var getAuth = new fmcAuthClass();
string authToken = getAuth.getFmcAuthToken(log);
if ("ERROR" == authToken)
{
log.LogError("AutoScaleManager:::: Failed to connect to FMC..Can not continue");
return((ActionResult) new BadRequestObjectResult("ERROR: Failed to connet to FMC..Can not continue"));
}
log.LogInformation("AutoScaleManager:::: Sampling Resource Utilization at {0}min Average", sampleTimeMin);
var factory = new AzureCredentialsFactory();
var msiCred = factory.FromMSI(new MSILoginInformation(MSIResourceType.AppService), AzureEnvironment.AzureGlobalCloud);
var azure = Azure.Configure().WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic).Authenticate(msiCred).WithSubscription(subscriptionId);
string resourceUri = null;
var vmss = azure.VirtualMachineScaleSets.GetByResourceGroup(resoureGroupName, vmScalesetName);
resourceUri = vmss.Id;
if (null == resourceUri)
{
log.LogError("AutoScaleManager:::: Unable to get resource uri");
return((ActionResult) new BadRequestObjectResult("ERROR: Unable to get resource uri"));
}
currentVmCapacity = vmss.Capacity;
log.LogWarning("AutoScaleManager:::: Current capacity of VMSS : {0}", currentVmCapacity);
//If the VMSS capacity is '0' consider this as first deployment and spawn 'minimum FTD count' at a time
if ((0 == currentVmCapacity) && (0 != minFTDCount))
{
log.LogWarning("AutoScaleManager:::: Current VMSS capacity is 0, considering it as first deployment (min FTD count needed : {0}", minFTDCount);
if ("BULK" == initialDeployMethod)
{
log.LogWarning("AutoScaleManager:::: Selected initial deployment mode is BULK");
log.LogWarning("AutoScaleManager:::: Deploying {0} number of FTDvs in scale set", minFTDCount);
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"" + minFTDCount + "\", \"TYPE\": \"INIT\" }";
return((ActionResult) new OkObjectResult(scaleStr));
}
else
{
log.LogWarning("AutoScaleManager:::: BULK method is not selected for initial deployment.. proceeding with STEP");
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"1\", \"TYPE\": \"REGULAR\"}";
return((ActionResult) new OkObjectResult(scaleStr));
}
}
//If current capacity is less than minimum FTD count requied then we need to scale-out
if (currentVmCapacity < minFTDCount)
{
log.LogWarning("AutoScaleManager:::: Current VMSS Capacity({0}) is less than minimum FTD count ({1}) needed.. time to SCALE-OUT", currentVmCapacity, minFTDCount);
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"1\", \"TYPE\": \"REGULAR\"}";
return((ActionResult) new OkObjectResult(scaleStr));
}
//-------------------------------------------------Scaling decission based on Metrics------------------------------------------------------
log.LogWarning("AutoScaleManager:::: Scaling Policy : {0}", scalingPolicy);
var sampleIntervalMin = System.TimeSpan.FromMinutes(Convert.ToDouble(sampleTimeMin));
MonitorManagementClient metricClient = new MonitorManagementClient(msiCred);
double ftdCpuUsage = 0;
double groupCpuUsage = 0;
double consolidatedCpuUsage = 0;
bool scaleInRejectFlag = false;
double minFtdCpuUsage = 9999;
string leastCpuLoadedFtd = "";
string leastCpuLoadedFtdIndex = "";
bool memoryMetricsEnabled = false;
double ftdMemUsage = 0;
double groupMemUsage = 0;
double consolidatedMemUsage = 0;
string ftdNameWithHighMemUtilization = "";
//Get FTD's Memory if 'Memory' metrics is enabled
if (metrics.Contains("memory"))
{
memoryMetricsEnabled = true;
log.LogInformation("Memory metrics enabled");
log.LogInformation("Memory Scale Out threshold: {0}%, Scale In threshold : {1}%", scaleOutThresholdMem, scaleInThresholdMem);
//Reject if Memory scale Out Threshold < scale In Threshold
if (scaleOutThresholdMem <= scaleInThresholdMem)
{
log.LogError("AutoScaleManager:::: Memory metrics ScaleOut Threshold ({0}) is less than or equal to ScaleIn Threshold ({1}) this is not correct", scaleOutThresholdMem, scaleInThresholdMem);
return((ActionResult) new BadRequestObjectResult("ERROR: Memory Metrics ScaleOut threshold is less than or equal to ScaleIn threshold"));
}
var getMetrics = new getMetricsClass();
var getId = new getDevIdByNameClass();
var devIds = getId.getAllDevId(authToken, log);
if ("ERROR" == devIds)
{
log.LogError("AutoScaleManager::::Unable to get device IDs");
return((ActionResult) new StatusCodeResult(StatusCodes.Status500InternalServerError));
}
//parse json object
JObject o = JObject.Parse(devIds);
foreach (var vm in vmss.VirtualMachines.List())
{
var vmName = vm.Name.ToString();
var devId = "";
try
{
foreach (var item in o["items"])
{
if (vmName == item["name"].ToString())
{
devId = item["id"].ToString();
break;
}
}
if (0 == devId.Length)
{
log.LogError("AutoScaleManager:::: Unable to get Device ID for Device Name({0})", vmName);
return((ActionResult) new StatusCodeResult(StatusCodes.Status500InternalServerError));
}
}
catch
{
log.LogError("AutoScaleManager:::: Exception Occoured while parsing device id response");
return((ActionResult) new StatusCodeResult(StatusCodes.Status500InternalServerError));
}
ftdMemUsage = Convert.ToDouble(getMetrics.getFtdMemoryMetrics(devId, authToken, log));
if (-1 == ftdMemUsage)
{
log.LogError("AutoScaleManager:::: Failed to get Memory usage of {0}", vmName);
return((ActionResult) new StatusCodeResult(StatusCodes.Status500InternalServerError));
}
if (ftdMemUsage > scaleInThresholdMem)
{
//No need to Scale-In
scaleInRejectFlag = true;
}
log.LogInformation("AutoScaleManager:::: Memory usage of {0} is {1} %", vmName, ftdMemUsage);
if ("POLICY-1" == scalingPolicy)
{
if (ftdMemUsage > scaleOutThresholdMem)
{
log.LogWarning("AutoScaleManager:::: FTD {0} has Memory Utilization of {1} % which is greater than Scale Out threshold", vmName, ftdMemUsage);
ftdNameWithHighMemUtilization = vmName;
break;
}
}
else if ("POLICY-2" == scalingPolicy)
{
groupMemUsage += ftdMemUsage;
}
}
groupMemUsage /= vmss.Capacity;
if ("POLICY-2" == scalingPolicy)
{
log.LogInformation("AutoScaleManager:::: Group Memory average usage : {0} %", groupMemUsage);
}
}
else
{
//Memory metrics not enabled, reset thresholds
scaleOutThresholdMem = 0;
}
if ("POLICY-2" == scalingPolicy)
{
log.LogInformation("AutoScaleManager:::: Scaling Policy-2 Selected..Getting average CPU utilization of scale set");
var response = await metricClient.Metrics.ListAsync(resourceUri, null, null, sampleIntervalMin, "Percentage CPU", "Average");
foreach (var metric in response.Value)
{
foreach (var series in metric.Timeseries)
{
foreach (var point in series.Data)
{
if (point.Average.HasValue)
{
groupCpuUsage = point.Average.Value;
log.LogDebug("AutoScaleManager:::: avg cpu: {0}", groupCpuUsage);
}
}
}
}
log.LogInformation("AutoScaleManager:::: Group CPU average usage : {0} %", groupCpuUsage);
}
foreach (var vm in vmss.VirtualMachines.List())
{
var vmName = vm.Name;
ftdCpuUsage = 0;
//Metrics filter
ODataQuery <MetadataValue> odataFilterMetrics = new ODataQuery <MetadataValue>(string.Format("VMName eq '{0}'", vmName));
// log.LogInformation("AutoScaleManager:::: Getting Metrics for : {0}", vmName);
var response = await metricClient.Metrics.ListAsync(resourceUri, odataFilterMetrics, null, sampleIntervalMin, "Percentage CPU", "Average");
foreach (var metric in response.Value)
{
foreach (var series in metric.Timeseries)
{
foreach (var point in series.Data)
{
if (point.Average.HasValue)
{
ftdCpuUsage = point.Average.Value;
log.LogDebug("AutoScaleManager:::: avg cpu: {0}", ftdCpuUsage);
}
}
}
}
log.LogInformation("AutoScaleManager:::: Avg CPU Utilizatio of VM({0}) in last {1}min : {2}%", vmName, sampleTimeMin, ftdCpuUsage);
//Maintain the FTD with minimum utilization to scale-in if needed
if (ftdCpuUsage < minFtdCpuUsage)
{
minFtdCpuUsage = ftdCpuUsage;
leastCpuLoadedFtd = vmName;
leastCpuLoadedFtdIndex = vm.InstanceId;
}
if ("POLICY-1" == scalingPolicy)
{
//Average usage of individual Instance
consolidatedCpuUsage = ftdCpuUsage;
consolidatedMemUsage = ftdMemUsage;
}
else if ("POLICY-2" == scalingPolicy)
{
//Scale Set average utilization
consolidatedCpuUsage = groupCpuUsage;
consolidatedMemUsage = groupMemUsage;
}
else
{
log.LogError("AutoScaleManager:::: Invalid Scaling Policy {0}", scalingPolicy);
return((ActionResult) new BadRequestObjectResult("ERROR: Invalid Scaling Policy"));
}
//If CPU utilization is greater than scale-out threshold then Scale-Out
//Note: if memory metrics is not enabled then consolidatedMemUsage will be always 0
if ((consolidatedCpuUsage > scaleOutThresholdCpu) || (consolidatedMemUsage > scaleOutThresholdMem))
{
//If current capacity is equal to max FTD count required then do nothing
//If current capacity is more than max FTD count (This should never happen) do nothing
if (currentVmCapacity >= maxFTDCount)
{
log.LogWarning("AutoScaleManager:::: Current VMSS Capacity({0}) is greater than or equal to max FTD count ({1}) needed.. No action needed", currentVmCapacity, maxFTDCount);
return((ActionResult) new OkObjectResult("NOACTION"));
}
if ("POLICY-1" == scalingPolicy)
{
log.LogWarning("AutoScaleManager:::: Avg CPU Utilizatio of VM({0}) in last {1}min is {2}% ", vmName, sampleTimeMin, consolidatedCpuUsage);
if (memoryMetricsEnabled && (consolidatedMemUsage > scaleOutThresholdMem))
{
log.LogWarning("AutoScaleManager:::: Avg Memory Utilizatio of VM({0}) is {1}% ", ftdNameWithHighMemUtilization, consolidatedMemUsage);
}
log.LogWarning("AutoScaleManager:::: Time to SCALE OUT");
}
else if ("POLICY-2" == scalingPolicy)
{
log.LogWarning("AutoScaleManager:::: Avg CPU Utilizatio of Scale Set in last {0}min is {1}% ", sampleTimeMin, consolidatedCpuUsage);
if (memoryMetricsEnabled)
{
log.LogWarning("AutoScaleManager:::: Avg Memory Utilizatio of Scale Set is {0}% ", consolidatedMemUsage);
}
log.LogWarning("AutoScaleManager:::: Average resource utilization of scale set is more than Scale Out threshold.. Time to SCALE OUT");
}
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"1\", \"TYPE\": \"REGULAR\" }";
return((ActionResult) new OkObjectResult(scaleStr));
}
//If any VM's CPU utilization is greater than scale-in threshold then Scale-In is not needed
else if (ftdCpuUsage > scaleInThresholdCpu)
{
scaleInRejectFlag = true;
}
}
//if scaleInRejectFlag is not set, it means all the VM's CPU & Memory utilization is less than or equal to Scale-In threshold
//Hence considering only least CPU consuming FTDv for Scale-In operation
if (false == scaleInRejectFlag)
{
//If current capacity is less than or equal to minimum FTD count requied then scale-in should not be done
if (currentVmCapacity <= minFTDCount)
{
log.LogWarning("AutoScaleManager:::: Scale-In needed but Current VMSS Capacity({0}) is less than or equal to minimum FTD count ({1}) needed.. No Action done", currentVmCapacity, minFTDCount);
return((ActionResult) new OkObjectResult("NOACTION"));
}
var networkInterfaceName = System.Environment.GetEnvironmentVariable("MNGT_NET_INTERFACE_NAME", EnvironmentVariableTarget.Process);
var ipConfigurationName = System.Environment.GetEnvironmentVariable("MNGT_IP_CONFIG_NAME", EnvironmentVariableTarget.Process);
var publicIpAddressName = System.Environment.GetEnvironmentVariable("MNGT_PUBLIC_IP_NAME", EnvironmentVariableTarget.Process);
var NmClient = new NetworkManagementClient(msiCred)
{
SubscriptionId = azure.SubscriptionId
};
var publicIp = NmClient.PublicIPAddresses.GetVirtualMachineScaleSetPublicIPAddress(resoureGroupName, vmScalesetName, leastCpuLoadedFtdIndex, networkInterfaceName, ipConfigurationName, publicIpAddressName).IpAddress;
log.LogWarning("AutoScaleManager:::: CPU Utilization of all the FTD's is less than or equal to CPU Scale-In threshold({0}%).. Time to SCALE-IN", scaleInThresholdCpu);
if (memoryMetricsEnabled)
{
log.LogWarning("AutoScaleManager:::: Memory Utilization of all the FTD's is less than or equal to Memory Scale-In threshold({0}%).. Time to SCALE-IN", scaleInThresholdMem);
}
log.LogWarning("AutoScaleManager:::: Least loaded FTD is : {0} with Utilization : {1}%", leastCpuLoadedFtd, minFtdCpuUsage);
scaleStr = "{ \"COMMAND\": \"SCALEIN\", \"ftdDevName\": \"" + leastCpuLoadedFtd + "\", \"ftdPublicIp\": \"" + publicIp + "\", \"instanceid\" : \"" + leastCpuLoadedFtdIndex + "\" }";
return((ActionResult) new OkObjectResult(scaleStr));
}
//Scaling not needed
log.LogWarning("AutoScaleManager:::: FTD scaleset utilization is within threshold.. no action needed");
return((ActionResult) new OkObjectResult("NOACTION"));
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogWarning("AutoScaleManager:::: Task to check Scaling requirement.. Started (ASM Version : V2.0)");
var resoureGroupName = System.Environment.GetEnvironmentVariable("RESOURCE_GROUP_NAME", EnvironmentVariableTarget.Process);
var vmScalesetName = System.Environment.GetEnvironmentVariable("VMSS_NAME", EnvironmentVariableTarget.Process);
var minFTDCountStr = System.Environment.GetEnvironmentVariable("MIN_FTD_COUNT", EnvironmentVariableTarget.Process);
var maxFTDCountStr = System.Environment.GetEnvironmentVariable("MAX_FTD_COUNT", EnvironmentVariableTarget.Process);
var sampleTimeMin = System.Environment.GetEnvironmentVariable("SAMPLING_TIME_MIN", EnvironmentVariableTarget.Process);
var scaleOutThresholdStr = System.Environment.GetEnvironmentVariable("SCALE_OUT_THRESHLD", EnvironmentVariableTarget.Process);
var scaleInThresholdStr = System.Environment.GetEnvironmentVariable("SCALE_IN_THRESHLD", EnvironmentVariableTarget.Process);
var initialDeployMethod = System.Environment.GetEnvironmentVariable("INITIAL_DEPLOYMENT_MODE", EnvironmentVariableTarget.Process); //supported STEP / BULK
var scalingPolicy = System.Environment.GetEnvironmentVariable("SCALING_POLICY", EnvironmentVariableTarget.Process); // POLICY-1 / POLICY-2
var subscriptionId = System.Environment.GetEnvironmentVariable("SUBSCRIPTION_ID", EnvironmentVariableTarget.Process);
int minFTDCount = Convert.ToInt32(minFTDCountStr);
int maxFTDCount = Convert.ToInt32(maxFTDCountStr);
double scaleOutThreshold = Convert.ToDouble(scaleOutThresholdStr);
double scaleInThreshold = Convert.ToDouble(scaleInThresholdStr);
int currentVmCapacity = 0;
string scaleStr = "";
//Reject if scaleOutThreshold < scaleInThreshold
if (scaleOutThreshold <= scaleInThreshold)
{
log.LogError("AutoScaleManager:::: ScaleOut Threshold ({0}) is less than or equal to ScaleIn Threshold ({1}) this is not correct", scaleOutThreshold, scaleInThreshold);
return((ActionResult) new BadRequestObjectResult("ERROR: ScaleOut threshold is less than or equal to ScaleIn threshold"));
}
//Check FMC connection, If we can not connect to FMC do not continue
log.LogInformation("AutoScaleManager:::: Checking FMC connection");
var getAuth = new fmcAuthClass();
string authToken = getAuth.getFmcAuthToken(log);
if ("ERROR" == authToken)
{
log.LogError("AutoScaleManager:::: Failed to connect to FMC..Can not continue");
return((ActionResult) new BadRequestObjectResult("ERROR: Failed to connet to FMC..Can not continue"));
}
log.LogInformation("AutoScaleManager:::: Sampling Resource Utilization at {0}min Average", sampleTimeMin);
var factory = new AzureCredentialsFactory();
var msiCred = factory.FromMSI(new MSILoginInformation(MSIResourceType.AppService), AzureEnvironment.AzureGlobalCloud);
var azure = Azure.Configure().WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic).Authenticate(msiCred).WithSubscription(subscriptionId);
string resourceUri = null;
var vmss = azure.VirtualMachineScaleSets.GetByResourceGroup(resoureGroupName, vmScalesetName);
resourceUri = vmss.Id;
if (null == resourceUri)
{
log.LogError("AutoScaleManager:::: Unable to get resource uri");
return((ActionResult) new BadRequestObjectResult("ERROR: Unable to get resource uri"));
}
currentVmCapacity = vmss.Capacity;
log.LogWarning("AutoScaleManager:::: Current capacity of VMSS : {0}", currentVmCapacity);
//If the VMSS capacity is '0' consider this as first deployment and spawn 'minimum FTD count' at a time
if ((0 == currentVmCapacity) && (0 != minFTDCount))
{
log.LogWarning("AutoScaleManager:::: Current VMSS capacity is 0, considering it as first deployment (min FTD count needed : {0}", minFTDCount);
if ("BULK" == initialDeployMethod)
{
log.LogWarning("AutoScaleManager:::: Selected initial deployment mode is BULK");
log.LogWarning("AutoScaleManager:::: Deploying {0} number of FTDvs in scale set", minFTDCount);
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"" + minFTDCount + "\", \"TYPE\": \"INIT\" }";
return((ActionResult) new OkObjectResult(scaleStr));
}
else
{
log.LogWarning("AutoScaleManager:::: BULK method is not selected for initial deployment.. proceeding with STEP");
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"1\", \"TYPE\": \"REGULAR\"}";
return((ActionResult) new OkObjectResult(scaleStr));
}
}
//If current capacity is less than minimum FTD count requied then we need to scale-out
if (currentVmCapacity < minFTDCount)
{
log.LogWarning("AutoScaleManager:::: Current VMSS Capacity({0}) is less than minimum FTD count ({1}) needed.. time to SCALE-OUT", currentVmCapacity, minFTDCount);
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"1\", \"TYPE\": \"REGULAR\"}";
return((ActionResult) new OkObjectResult(scaleStr));
}
//-------------------------------------------------Scaling decission based on Metrics------------------------------------------------------
var sampleIntervalMin = System.TimeSpan.FromMinutes(Convert.ToDouble(sampleTimeMin));
MonitorManagementClient metricClient = new MonitorManagementClient(msiCred);
double ftdUsage = 0;
double groupUsage = 0;
double consolidatedUsage = 0;
bool scaleInRejectFlag = false;
double minFtdUsage = 9999;
string leastLoadedFtd = "";
string leastLoadedFtdIndex = "";
log.LogWarning("AutoScaleManager:::: Scaling Policy : {0}", scalingPolicy);
if ("POLICY-2" == scalingPolicy)
{
log.LogInformation("AutoScaleManager:::: Scaling Policy-2 Selected..Getting average CPU utilization of scale set");
var response = await metricClient.Metrics.ListAsync(resourceUri, null, null, sampleIntervalMin, "Percentage CPU", "Average");
foreach (var metric in response.Value)
{
foreach (var series in metric.Timeseries)
{
foreach (var point in series.Data)
{
if (point.Average.HasValue)
{
groupUsage = point.Average.Value;
log.LogDebug("AutoScaleManager:::: avg cpu: {0}", ftdUsage);
}
}
}
}
log.LogInformation("AutoScaleManager:::: Group average usage : {0}", groupUsage);
}
foreach (var vm in vmss.VirtualMachines.List())
{
var vmName = vm.Name;
ftdUsage = 0;
//Metrics filter
ODataQuery <MetadataValue> odataFilterMetrics = new ODataQuery <MetadataValue>(string.Format("VMName eq '{0}'", vmName));
log.LogInformation("AutoScaleManager:::: Getting Metrics for : {0}", vmName);
var response = await metricClient.Metrics.ListAsync(resourceUri, odataFilterMetrics, null, sampleIntervalMin, "Percentage CPU", "Average");
foreach (var metric in response.Value)
{
foreach (var series in metric.Timeseries)
{
foreach (var point in series.Data)
{
if (point.Average.HasValue)
{
ftdUsage = point.Average.Value;
log.LogDebug("AutoScaleManager:::: avg cpu: {0}", ftdUsage);
}
}
}
}
log.LogInformation("AutoScaleManager:::: Avg CPU Utilizatio of VM({0}) in last {1}min : {2}%", vmName, sampleTimeMin, ftdUsage);
//Maintain the FTD with minimum utilization to scale-in if needed
if (ftdUsage < minFtdUsage)
{
minFtdUsage = ftdUsage;
leastLoadedFtd = vmName;
leastLoadedFtdIndex = vm.InstanceId;
}
if ("POLICY-1" == scalingPolicy)
{
//Average usage of individual Instance
consolidatedUsage = ftdUsage;
}
else if ("POLICY-2" == scalingPolicy)
{
//Scale Set average utilization
consolidatedUsage = groupUsage;
}
else
{
log.LogError("Invalid Scaling Policy {0}", scalingPolicy);
return((ActionResult) new BadRequestObjectResult("ERROR: Invalid Scaling Policy"));
}
//If CPU utilization is greater than scale-out threshold then Scale-Out
if (consolidatedUsage > scaleOutThreshold)
{
//If current capacity is equal to max FTD count required then do nothing
//If current capacity is more than max FTD count (This should never happen) do nothing
if (currentVmCapacity >= maxFTDCount)
{
log.LogWarning("AutoScaleManager:::: Current VMSS Capacity({0}) is greater than or equal to max FTD count ({1}) needed.. No action needed", currentVmCapacity, maxFTDCount);
return((ActionResult) new OkObjectResult("NOACTION"));
}
if ("POLICY-1" == scalingPolicy)
{
log.LogWarning("AutoScaleManager:::: Avg CPU Utilizatio of VM({0}) in last {1}min is {2}% which is greater than ScaleOut threshold({3}%) .. Time to SCALE-OUT", vmName, sampleTimeMin, consolidatedUsage, scaleOutThreshold);
}
else if ("POLICY-2" == scalingPolicy)
{
log.LogWarning("AutoScaleManager:::: Avg CPU Utilizatio of Scale Set in last {0}min is {1}% which is greater than ScaleOut threshold({2}%) .. Time to SCALE-OUT", sampleTimeMin, consolidatedUsage, scaleOutThreshold);
}
scaleStr = "{ \"COMMAND\": \"SCALEOUT\", \"COUNT\": \"1\", \"TYPE\": \"REGULAR\" }";
return((ActionResult) new OkObjectResult(scaleStr));
}
//If any VM's CPU utilization is greater than scale-in threshold then Scale-In is not needed
else if (ftdUsage > scaleInThreshold)
{
scaleInRejectFlag = true;
}
}
//if scaleInRejectFlag is not set, it means all the VM's CPU utilization is less than or equal to Scale-In threshold
if (false == scaleInRejectFlag)
{
//If current capacity is less than or equal to minimum FTD count requied then scale-in should not be done
if (currentVmCapacity <= minFTDCount)
{
log.LogWarning("AutoScaleManager:::: Scale-In needed but Current VMSS Capacity({0}) is less than or equal to minimum FTD count ({1}) needed.. No Action done", currentVmCapacity, minFTDCount);
return((ActionResult) new OkObjectResult("NOACTION"));
}
var networkInterfaceName = System.Environment.GetEnvironmentVariable("MNGT_NET_INTERFACE_NAME", EnvironmentVariableTarget.Process);
var ipConfigurationName = System.Environment.GetEnvironmentVariable("MNGT_IP_CONFIG_NAME", EnvironmentVariableTarget.Process);
var publicIpAddressName = System.Environment.GetEnvironmentVariable("MNGT_PUBLIC_IP_NAME", EnvironmentVariableTarget.Process);
var NmClient = new NetworkManagementClient(msiCred)
{
SubscriptionId = azure.SubscriptionId
};
var publicIp = NmClient.PublicIPAddresses.GetVirtualMachineScaleSetPublicIPAddress(resoureGroupName, vmScalesetName, leastLoadedFtdIndex, networkInterfaceName, ipConfigurationName, publicIpAddressName).IpAddress;
log.LogWarning("AutoScaleManager:::: CPU Utilization of all the FTD's is less than or equal to Scale-In threshold({0}%).. Time to SCALE-IN", scaleInThreshold);
log.LogWarning("AutoScaleManager:::: Least loaded FTD is : {0} with Utilization : {1}%", leastLoadedFtd, minFtdUsage);
scaleStr = "{ \"COMMAND\": \"SCALEIN\", \"ftdDevName\": \"" + leastLoadedFtd + "\", \"ftdPublicIp\": \"" + publicIp + "\", \"instanceid\" : \"" + leastLoadedFtdIndex + "\" }";
return((ActionResult) new OkObjectResult(scaleStr));
}
//Scaling not needed
log.LogWarning("AutoScaleManager:::: FTD scaleset utilization is within threshold.. no action needed");
return((ActionResult) new OkObjectResult("NOACTION"));
}