/// <summary>Include the virtual machine batch operation into existing batch opeation</summary>
/// <param name="virtualMachineConcurrentBag"></param>
/// <param name="batchTasks"></param>
/// <param name="log"></param>
private static void IncludeVirtualMachineTask(ConcurrentBag <IEnumerable <IGrouping <string, IVirtualMachine> > > virtualMachineConcurrentBag,
ConcurrentBag <Task> batchTasks, TraceWriter log)
{
var groupsByVirtulaMachine = virtualMachineConcurrentBag.SelectMany(x => x);
var virtualmachineCloudTable = StorageAccountProvider.CreateOrGetTable(StorageTableNames.VirtualMachineCrawlerTableName);
if (virtualmachineCloudTable == null)
{
return;
}
Parallel.ForEach(groupsByVirtulaMachine, groupItem =>
{
// table batch operation currently allows only 100 per batch, So ensuring the one batch operation will have only 100 items
for (var i = 0; i < groupItem.Count(); i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = groupItem.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
var virtualMachineBatchOperation = GetVirtualMachineBatchOperation(batchItems, groupItem.Key, log);
if (virtualMachineBatchOperation != null && virtualMachineBatchOperation.Count > 0 && virtualmachineCloudTable != null)
{
batchTasks.Add(virtualmachineCloudTable.ExecuteBatchAsync(virtualMachineBatchOperation));
}
}
});
}
public void CreateRule(TraceWriter log)
{
try
{
var loadBalancer = GetRandomLoadBalancer();
if (loadBalancer == null)
{
log.Info("Loadbalancer RuleEngine: No load balancer found with virtual machines.");
return;
}
var filteredVmSet = GetVirtualMachineSet(loadBalancer.Id);
if (filteredVmSet == null)
{
log.Info("Loadbalancer RuleEngine: No virtual machines found for the load balancer name: " + loadBalancer.ResourceName);
return;
}
var table = StorageAccountProvider.CreateOrGetTable(StorageTableNames.ScheduledRulesTableName);
if (table == null)
{
return;
}
var count = VmCount(filteredVmSet.Count);
var tasks = new List <Task>();
// do
// {
var randomSets = filteredVmSet.Take(count).ToList();
filteredVmSet = filteredVmSet.Except(randomSets).ToList();
for (var i = 0;
i < randomSets.Count;
i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = randomSets.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
var batchOperation = VirtualMachineHelper.CreateScheduleEntity(batchItems,
_azureClient.AzureSettings.Chaos.SchedulerFrequency,
_azureClient.AzureSettings.Chaos.AzureFaultInjectionActions,
VirtualMachineGroup.LoadBalancer);
var operation = batchOperation;
tasks.Add(table.ExecuteBatchAsync(operation));
}
// } while (filteredVmSet.Any());
Task.WhenAll(tasks);
log.Info("Loadbalancer RuleEngine: Completed creating rule engine.");
}
catch (Exception ex)
{
log.Error("LoadBalancer RuleEngine: Exception thrown. ", ex);
}
}
private void InsertVirtualMachineAvailabilitySetDomainResults(string availabilitySetId, int domainNumber)
{
var virtualMachineQuery = TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("AvailabilitySetId",
QueryComparisons.Equal,
availabilitySetId),
TableOperators.And,
_azureClient.AzureSettings.Chaos.AvailabilitySetChaos.FaultDomainEnabled
? TableQuery.GenerateFilterConditionForInt("FaultDomain",
QueryComparisons.Equal,
domainNumber)
: TableQuery.GenerateFilterConditionForInt("UpdateDomain",
QueryComparisons.Equal,
domainNumber));
//TableQuery.GenerateFilterConditionForInt("AvailabilityZone", QueryComparisons.GreaterThanOrEqual, 0);
var virtualMachinesTableQuery = new TableQuery <VirtualMachineCrawlerResponse>().Where(virtualMachineQuery);
var crawledVirtualMachinesResults = StorageAccountProvider.GetEntities(virtualMachinesTableQuery,
StorageTableNames.VirtualMachineCrawlerTableName);
var virtualMachinesResults = crawledVirtualMachinesResults.ToList();
if (!virtualMachinesResults.Any())
{
return;
}
var domainFlag = !_azureClient.AzureSettings.Chaos.AvailabilitySetChaos.UpdateDomainEnabled;
var batchTasks = new List <Task>();
var table = StorageAccountProvider.CreateOrGetTable(StorageTableNames.ScheduledRulesTableName);
if (table == null)
{
return;
}
for (var i = 0; i < virtualMachinesResults.Count; i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = virtualMachinesResults.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
var scheduledRulesbatchOperation =
VirtualMachineHelper.CreateScheduleEntityForAvailabilitySet(batchItems,
_azureClient.AzureSettings.Chaos.SchedulerFrequency,
_azureClient.AzureSettings.Chaos.AzureFaultInjectionActions,
domainFlag);
if (scheduledRulesbatchOperation.Count <= 0)
{
return;
}
batchTasks.Add(table.ExecuteBatchAsync(scheduledRulesbatchOperation));
}
if (batchTasks.Count > 0)
{
Task.WhenAll(batchTasks);
}
}
/// <summary>Get the list of the availability sets by resource group.
/// And get the virtual machine by resource group and the availability sets.
/// And get the batch operation for the availability sets</summary>
/// <param name="resourceGroupList"></param>
/// <param name="virtualMachinesConcurrent"></param>
/// <param name="batchTasks"></param>
/// <param name="azureClient"></param>
/// <param name="log"></param>
private static void SetTheVirtualMachinesAndAvailabilitySetBatchTask(IEnumerable <IResourceGroup> resourceGroupList,
ConcurrentBag <IEnumerable <IGrouping <string, IVirtualMachine> > > virtualMachinesConcurrent,
ConcurrentBag <Task> batchTasks,
AzureClient azureClient,
TraceWriter log)
{
var availabilitySetCloudTable = StorageAccountProvider.CreateOrGetTable(StorageTableNames.AvailabilitySetCrawlerTableName);
if (availabilitySetCloudTable == null)
{
return;
}
Parallel.ForEach(resourceGroupList, eachResourceGroup =>
{
try
{
var availabilitySetIds = new List <string>();
var availabilitySetsByResourceGroup = azureClient.AzureInstance.AvailabilitySets.ListByResourceGroup(eachResourceGroup.Name);
// table batch operation currently allows only 100 per batch, So ensuring the one batch operation will have only 100 items
var setsByResourceGroup = availabilitySetsByResourceGroup.ToList();
for (var i = 0; i < setsByResourceGroup.Count; i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = setsByResourceGroup.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
// get the availability sets by resource group
// get the availability sets batch operation and get the list of availability set ids
var availabilitySetbatchOperation =
GetAvailabilitySetBatchOperation(batchItems, availabilitySetIds);
// add the batch operation into task list
if (availabilitySetbatchOperation.Count > 0 && availabilitySetCloudTable != null)
{
batchTasks.Add(availabilitySetCloudTable.ExecuteBatchAsync(availabilitySetbatchOperation));
}
}
// Get the virtual machines by resource group and by availability set ids
var virtualMachinesByAvailabilitySetId = GetVirtualMachineListByResourceGroup(eachResourceGroup.Name, availabilitySetIds, azureClient);
if (virtualMachinesByAvailabilitySetId != null && virtualMachinesByAvailabilitySetId.Count > 0)
{
virtualMachinesConcurrent.Add(virtualMachinesByAvailabilitySetId);
}
}
catch (Exception e)
{
log.Error($"timercrawlerforavailableset threw the exception ", e,
"for resource group: " + eachResourceGroup.Name);
}
});
}
/// <summary>Create the virtual machine rules</summary>
/// <param name="log"></param>
public void CreateRule(TraceWriter log)
{
try
{
log.Info("VirtualMachine RuleEngine: Started the creating rules for the virtual machines.");
var vmSets = GetRandomVmSet();
if (vmSets == null)
{
log.Info("VirtualMachine RuleEngine: No virtual machines found..");
return;
}
var table = StorageAccountProvider.CreateOrGetTable(StorageTableNames.ScheduledRulesTableName);
var count = VmCount(vmSets.Count);
var tasks = new List <Task>();
//do
//{
var randomSets = vmSets.Take(count).ToList();
vmSets = vmSets.Except(randomSets).ToList();
for (var i = 0;
i < randomSets.Count;
i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = randomSets.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
var batchOperation = VirtualMachineHelper.CreateScheduleEntity(batchItems,
azureClient.AzureSettings.Chaos.SchedulerFrequency,
azureClient.AzureSettings.Chaos.AzureFaultInjectionActions,
VirtualMachineGroup.VirtualMachines);
if (batchOperation == null)
{
continue;
}
tasks.Add(table.ExecuteBatchAsync(batchOperation));
}
// } while (vmSets.Any());
Task.WhenAll(tasks);
log.Info("VirtualMachine RuleEngine: Completed creating rule engine..");
}
catch (Exception ex)
{
log.Error("VirtualMachine RuleEngine: Exception thrown. ", ex);
}
}
/// <summary>1. Iterate the resource groups to get the scale sets for individual resource group.
/// 2. Convert the List of scale sets into scale set entity and add them into the table batch operation.
/// 3. Get the list of virtual machine instances, convert into entity and them into the table batach operation
/// 3. Execute all the task parallely</summary>
/// <param name="resourceGroups">List of resource groups for the particular subscription.</param>
/// <param name="log">Trace writer instance</param>
private static async Task GetLoadBalancersForResourceGroupsAsync(IEnumerable <IResourceGroup> resourceGroups,
TraceWriter log)
{
try
{
var virtualMachineCloudTable = StorageAccountProvider.CreateOrGetTable(StorageTableNames.VirtualMachineCrawlerTableName);
var loadBalancerTable = StorageAccountProvider.CreateOrGetTable(StorageTableNames.LoadBalancerCrawlerTableName);
if (virtualMachineCloudTable == null || loadBalancerTable == null)
{
return;
}
var batchTasks = new ConcurrentBag <Task>();
var azureClient = new AzureClient();
// using parallel here to run all the resource groups parallelly, parallel is 10times faster than normal foreach.
foreach (var eachResourceGroup in resourceGroups)
//Parallel.ForEach(resourceGroups, eachResourceGroup =>
{
try
{
var loadBalancersList = azureClient.AzureInstance.LoadBalancers
.ListByResourceGroup(eachResourceGroup.Name);
//var count = loadBalancersList.Count();
if (loadBalancersList.Count() > 0)
{
GetVirtualMachineAndLoadBalancerBatch(loadBalancersList.ToList(), batchTasks,
virtualMachineCloudTable,
loadBalancerTable, azureClient, log);
}
}
catch (Exception e)
{
// catch the error, to continue adding other entities to table
log.Error($"timercrawlerforloadbalancer threw the exception ", e,
"GetLoadBalancerForResourceGroups: for the resource group " + eachResourceGroup.Name);
}
}
//);
// execute all batch operation as parallel
await Task.WhenAll(batchTasks);
}
catch (Exception ex)
{
log.Error($"timercrawlerforloadbalancer threw the exception ", ex, "GetLoadBalancerForResourceGroups");
}
}
/// <summary>1. Iterate the resource groups to get the virtual machines for individual resource group.
/// 2. Convert the List of virtual machines into entities and add them into tasklist, repeat this process for the VM's under the resource group.
/// 3. Execute all the task parallely</summary>
/// <param name="resourceGroups">List of resource groups for the particular subscription.</param>
/// <param name="log">Trace writer instance</param>
private static async Task GetVirtualMachinesByResourceGroups(IEnumerable <IResourceGroup> resourceGroups,
TraceWriter log)
{
try
{
var virtualMachineCloudTable = StorageAccountProvider.CreateOrGetTable(StorageTableNames.VirtualMachineCrawlerTableName);
if (virtualMachineCloudTable == null)
{
return;
}
var batchTasks = new ConcurrentBag <Task>();
// using parallel here to run all the resource groups parallelly, parallel is 10times faster than normal foreach.
Parallel.ForEach(resourceGroups, async eachResourceGroup =>
{
var virtualMachinesByResourceGroup = await GetVirtualMachinesByResourceGroup(eachResourceGroup, log);
if (virtualMachinesByResourceGroup == null)
{
return;
}
var virtualMachineList = virtualMachinesByResourceGroup.ToList();
// table batch operation currently allows only 100 per batch, So ensuring the one batch operation will have only 100 items
for (var i = 0; i < virtualMachineList.Count(); i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = virtualMachineList.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
var batchOperation = InsertOrReplaceEntitiesIntoTable(batchItems,
eachResourceGroup.Name, log);
if (batchOperation != null)
{
batchTasks.Add(virtualMachineCloudTable.ExecuteBatchAsync(batchOperation));
}
}
});
await Task.WhenAll(batchTasks);
}
catch (Exception e)
{
log.Error($"timercrawlerforvirtualmachines:threw exception", e, "GetVirtualMachinesForResourceGroups");
}
}
private static async Task InsertOrReplaceResourceGroupsAsync(List <IResourceGroup> resourceGroups, TraceWriter log)
{
var tableBatchOperation = new TableBatchOperation();
// table batch operation currently allows only 100 per batch, So ensuring the one batch operation will have only 100 items
for (var i = 0; i < resourceGroups.Count; i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = resourceGroups.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
foreach (var eachResourceGroup in batchItems)
{
var resourceGroupCrawlerResponseEntity =
new ResourceGroupCrawlerResponse("", eachResourceGroup.Name);
try
{
resourceGroupCrawlerResponseEntity.Id = eachResourceGroup.Id;
resourceGroupCrawlerResponseEntity.RegionName = eachResourceGroup.RegionName;
tableBatchOperation.InsertOrReplace(resourceGroupCrawlerResponseEntity);
}
catch (Exception ex)
{
resourceGroupCrawlerResponseEntity.Error = ex.Message;
log.Error($"timercrawlerresourcegroups threw exception ", ex, "timercrawlerresourcegroups");
}
}
}
if (tableBatchOperation.Count > 0)
{
try
{
var table = StorageAccountProvider.CreateOrGetTable(StorageTableNames.ResourceGroupCrawlerTableName);
await table.ExecuteBatchAsync(tableBatchOperation);
}
catch (Exception ex)
{
log.Error($"timercrawlerresourcegroups threw exception ", ex, "timercrawlerresourcegroups");
}
}
}
private void InsertVirtualMachineAvailabilityZoneRegionResults(string region, int availbilityZone, TraceWriter log)
{
try
{
var virtualMachineQuery = TableQuery.CombineFilters(TableQuery.GenerateFilterConditionForInt(
"AvailabilityZone",
QueryComparisons.Equal,
availbilityZone),
TableOperators.And,
TableQuery.GenerateFilterCondition("RegionName",
QueryComparisons.Equal,
region));
//TableQuery.GenerateFilterConditionForInt("AvailabilityZone", QueryComparisons.GreaterThanOrEqual, 0);
var virtualMachinesTableQuery = new TableQuery <VirtualMachineCrawlerResponse>().Where(virtualMachineQuery);
var crawledVirtualMachinesResults = StorageAccountProvider.GetEntities(
virtualMachinesTableQuery,
StorageTableNames.VirtualMachineCrawlerTableName);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
crawledVirtualMachinesResults = crawledVirtualMachinesResults.Where(x => PowerState.Parse(x.State) == PowerState.Running);
var virtualMachinesResults = crawledVirtualMachinesResults.ToList();
var meanTimeQuery = TableQuery.GenerateFilterConditionForDate("ScheduledExecutionTime",
QueryComparisons.GreaterThanOrEqual,
DateTimeOffset.UtcNow.AddMinutes(-azureClient.AzureSettings.Chaos.MeanTime));
var recentlyExecutedAvailabilityZoneRegionCombinationQuery = TableQuery.GenerateFilterCondition(
"ResourceType",
QueryComparisons.Equal,
VirtualMachineGroup.VirtualMachineScaleSets.ToString());
var recentlyExecutedVMScaleSetsQuery = TableQuery.CombineFilters(meanTimeQuery,
TableOperators.And,
recentlyExecutedAvailabilityZoneRegionCombinationQuery);
var scheduledQuery = new TableQuery <ScheduledRules>().Where(recentlyExecutedVMScaleSetsQuery);
//Will get the executed query results.
var executedVMScaleSetsResults = StorageAccountProvider.GetEntities(scheduledQuery,
StorageTableNames.ScheduledRulesTableName);
List <VirtualMachineCrawlerResponse> executedVMAvailabilityZones = null;
if (virtualMachinesResults.Count() != 0 && executedVMScaleSetsResults.Count() != 0)
{
foreach (var virtualMachineResult in virtualMachinesResults)
{
foreach (var executedVMScaleSetsResult in executedVMScaleSetsResults)
{
if ((executedVMScaleSetsResult.RowKey.ToLowerInvariant().Contains(virtualMachineResult.ResourceName.ToLowerInvariant())) &&
(executedVMScaleSetsResult.RowKey.ToLowerInvariant().Contains(virtualMachineResult.ResourceGroupName.ToLowerInvariant())) &&
(executedVMScaleSetsResult.RowKey.ToLowerInvariant().Contains(virtualMachineResult.AvailabilityZone.ToString()))
//&& (executedVMScaleSetsResult.RowKey.ToLowerInvariant().Contains(executedVMScaleSetsResult..ToLowerInvariant()))
)
{
executedVMAvailabilityZones.Add(virtualMachineResult);
}
}
}
if (executedVMAvailabilityZones != null && executedVMAvailabilityZones.Count() != 0)
{
virtualMachinesResults = virtualMachinesResults.Except(executedVMAvailabilityZones).ToList();
}
}
if (!virtualMachinesResults.Any())
{
return;
}
var batchTasks = new List <Task>();
var table = StorageAccountProvider.CreateOrGetTable(StorageTableNames.ScheduledRulesTableName);
if (table == null)
{
return;
}
for (var i = 0; i < virtualMachinesResults.Count; i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = virtualMachinesResults.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
var scheduledRulesbatchOperation = VirtualMachineHelper
.CreateScheduleEntityForAvailabilityZone(
batchItems,
azureClient.AzureSettings.Chaos.SchedulerFrequency,
azureClient.AzureSettings.Chaos.AzureFaultInjectionActions);
if (scheduledRulesbatchOperation.Count <= 0)
{
return;
}
batchTasks.Add(table.ExecuteBatchAsync(scheduledRulesbatchOperation));
}
if (batchTasks.Count > 0)
{
Task.WhenAll(batchTasks);
}
}
catch (Exception ex)
{
log.Error("AvailabilityZone RuleEngine: thrown exception", ex);
log.Error(ex.Source);
log.Error(ex.Message);
}
}