/// <summary>Include the virtual machine batch operation into existing batch opeation</summary>
/// <param name="virtualMachineConcurrentBag"></param>
/// <param name="batchTasks"></param>
/// <param name="log"></param>
private static void IncludeVirtualMachineTask(ConcurrentBag <IEnumerable <IGrouping <string, IVirtualMachine> > > virtualMachineConcurrentBag,
ConcurrentBag <Task> batchTasks, TraceWriter log)
{
var groupsByVirtulaMachine = virtualMachineConcurrentBag.SelectMany(x => x);
var virtualmachineCloudTable = StorageAccountProvider.CreateOrGetTable(StorageTableNames.VirtualMachineCrawlerTableName);
if (virtualmachineCloudTable == null)
{
return;
}
Parallel.ForEach(groupsByVirtulaMachine, groupItem =>
{
// table batch operation currently allows only 100 per batch, So ensuring the one batch operation will have only 100 items
for (var i = 0; i < groupItem.Count(); i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = groupItem.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
var virtualMachineBatchOperation = GetVirtualMachineBatchOperation(batchItems, groupItem.Key, log);
if (virtualMachineBatchOperation != null && virtualMachineBatchOperation.Count > 0 && virtualmachineCloudTable != null)
{
batchTasks.Add(virtualmachineCloudTable.ExecuteBatchAsync(virtualMachineBatchOperation));
}
}
});
}
/// <summary>
/// ドメインサービスを DI に登録します。
/// </summary>
/// <param name="services"></param>
/// <param name="appSettings"></param>
/// <returns></returns>
public static IServiceCollection AddDomainServices(this IServiceCollection services, AppSettings appSettings)
{
var storageAccount = new StorageAccountProvider(appSettings);
services.AddInstagramSearch(storageAccount.AzureWebJobs, appSettings.CognitiveSearch, appSettings.WebProxy);
return(services);
}
public Function(
IHttpClientFactory httpClientFactory,
StorageAccountProvider storageProvider)
{
_httpClientFactory = httpClientFactory;
_storageProvider = storageProvider;
}
private static IEnumerable <ScheduledRules> GetScheduledRulesForRollback(TraceWriter log)
{
try
{
var azureSettings = new AzureClient().AzureSettings;
var dateFilterByUtc = TableQuery.GenerateFilterConditionForDate("EventCompletedTime",
QueryComparisons.LessThanOrEqual,
DateTimeOffset.UtcNow.AddMinutes(-azureSettings.Chaos.RollbackRunFrequency));
var dateFilterByFrequency = TableQuery.GenerateFilterConditionForDate("EventCompletedTime",
QueryComparisons.GreaterThanOrEqual,
DateTimeOffset.UtcNow.AddMinutes(-azureSettings.Chaos.RollbackRunFrequency - TriggerFrequency));
var statusFilter =
TableQuery.GenerateFilterCondition("ExecutionStatus",
QueryComparisons.Equal,
Status.Completed.ToString());
var rollbackFilter =
TableQuery.GenerateFilterConditionForBool("Rolledback", QueryComparisons.Equal, false);
var rollbackStatusFilter = TableQuery.CombineFilters(statusFilter, TableOperators.And, rollbackFilter);
var filter = TableQuery.CombineFilters(dateFilterByUtc, TableOperators.And, dateFilterByFrequency);
var scheduledQuery = new TableQuery <ScheduledRules>().Where(TableQuery.CombineFilters(filter,
TableOperators.And,
rollbackStatusFilter));
return(StorageAccountProvider.GetEntities(scheduledQuery, StorageTableNames.ScheduledRulesTableName));
}
catch (Exception e)
{
log.Error($"TimerTrigger function threw exception", e, "GetScheduledRulesForRollback");
throw;
}
}
public BlobTriggerAttributeBindingProvider(INameResolver nameResolver,
StorageAccountProvider accountProvider,
IHostIdProvider hostIdProvider,
IOptions <QueuesOptions> queueOptions,
IOptions <BlobsOptions> blobsOptions,
IWebJobsExceptionHandler exceptionHandler,
IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter,
SharedQueueWatcher messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
IHostSingletonManager singletonManager,
ILoggerFactory loggerFactory)
{
_accountProvider = accountProvider ?? throw new ArgumentNullException(nameof(accountProvider));
_hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider));
_queueOptions = (queueOptions ?? throw new ArgumentNullException(nameof(queueOptions))).Value;
_blobsOptions = (blobsOptions ?? throw new ArgumentNullException(nameof(blobsOptions))).Value;
_exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler));
_blobWrittenWatcherSetter = blobWrittenWatcherSetter ?? throw new ArgumentNullException(nameof(blobWrittenWatcherSetter));
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter));
_sharedContextProvider = sharedContextProvider ?? throw new ArgumentNullException(nameof(sharedContextProvider));
_singletonManager = singletonManager ?? throw new ArgumentNullException(nameof(singletonManager));
_nameResolver = nameResolver;
_loggerFactory = loggerFactory;
}
private static IHost CreateConfiguration <TResult>(StorageAccount storageAccount, Type programType,
IJobActivator activator, TaskCompletionSource <TResult> taskSource,
IFunctionInstanceLogger functionInstanceLogger, IExtensionRegistry extensions = null)
{
StorageAccountProvider storageAccountProvider = null; // new FakeStorageAccountProvider(storageAccount); $$$
IWebJobsExceptionHandler exceptionHandler = new TaskBackgroundExceptionHandler <TResult>(taskSource);
return(new HostBuilder()
.ConfigureDefaultTestHost(b =>
{
b.UseHostId(Guid.NewGuid().ToString("N"))
.AddAzureStorage();
}, programType)
.ConfigureServices(services =>
{
// services.AddSingleton<IOptionsFactory<JobHostQueuesOptions>, FakeQueuesOptionsFactory>(); $$$ ???
services.AddSingletonIfNotNull(storageAccountProvider);
services.AddSingletonIfNotNull(activator);
services.AddSingletonIfNotNull(exceptionHandler);
services.AddSingletonIfNotNull(extensions);
services.AddSingleton <IFunctionInstanceLoggerProvider>(new NullFunctionInstanceLoggerProvider(functionInstanceLogger));
services.AddSingleton <IHostInstanceLoggerProvider>(new NullHostInstanceLoggerProvider());
services.AddSingleton <IFunctionOutputLoggerProvider>(new NullFunctionOutputLoggerProvider());
})
.Build());
}
private List <string> GetAllPossibleAvailabilityZoneRegionCombination()
{
var possibleAvailabilityZoneRegionCombinationVmCount = new Dictionary <string, int>();
var crawledAvailabilityZoneVmQuery =
TableQuery.GenerateFilterConditionForInt("AvailabilityZone", QueryComparisons.GreaterThan, 0);
var availabilityZoneTableQuery = new TableQuery <VirtualMachineCrawlerResponse>().Where(crawledAvailabilityZoneVmQuery);
var crawledVirtualMachinesResults = StorageAccountProvider.GetEntities(availabilityZoneTableQuery,
StorageTableNames.VirtualMachineCrawlerTableName);
crawledVirtualMachinesResults = crawledVirtualMachinesResults.Where(x => PowerState.Parse(x.State) == PowerState.Running);
foreach (var eachCrawledVirtualMachinesResult in crawledVirtualMachinesResults)
{
var entryIntoPossibleAvailabilityZoneRegionCombinationVmCount =
eachCrawledVirtualMachinesResult.RegionName +
"!" +
eachCrawledVirtualMachinesResult.AvailabilityZone.ToString();
if (possibleAvailabilityZoneRegionCombinationVmCount.ContainsKey(entryIntoPossibleAvailabilityZoneRegionCombinationVmCount))
{
possibleAvailabilityZoneRegionCombinationVmCount[entryIntoPossibleAvailabilityZoneRegionCombinationVmCount] += 1;
}
else
{
possibleAvailabilityZoneRegionCombinationVmCount[entryIntoPossibleAvailabilityZoneRegionCombinationVmCount] = 1;
}
}
return(new List <string>(possibleAvailabilityZoneRegionCombinationVmCount.Keys));
}
public QueuesExtensionConfigProvider(StorageAccountProvider storageAccountProvider, IContextGetter <IMessageEnqueuedWatcher> contextGetter,
QueueTriggerAttributeBindingProvider triggerProvider)
{
_contextGetter = contextGetter;
_storageAccountProvider = storageAccountProvider;
_triggerProvider = triggerProvider;
}
private List <string> GetPossibleAvailabilitySets()
{
string availabilitySetQuery = TableQuery.GenerateFilterConditionForBool("HasVirtualMachines", QueryComparisons.Equal, true);
var availabilitySetTableQuery = new TableQuery <AvailabilitySetsCrawlerResponse>().Where(availabilitySetQuery);
var crawledAvailabilitySetResults = StorageAccountProvider.GetEntities(availabilitySetTableQuery, StorageTableNames.AvailabilitySetCrawlerTableName);
if (crawledAvailabilitySetResults == null)
{
return(null);
}
Dictionary <string, int> possibleAvailabilitySetDomainCombinationVmCount = new Dictionary <string, int>();
var bootStrapQuery = string.Empty;
var initialQuery = true;
foreach (var eachAvailabilitySet in crawledAvailabilitySetResults)
{
if (initialQuery)
{
bootStrapQuery = TableQuery.GenerateFilterCondition("AvailabilitySetId", QueryComparisons.Equal, ConvertToProperAvailableSetId(eachAvailabilitySet.RowKey));
initialQuery = false;
}
else
{
var localAvailabilitySetQuery = TableQuery.GenerateFilterCondition("AvailabilitySetId", QueryComparisons.Equal, ConvertToProperAvailableSetId(eachAvailabilitySet.RowKey));
bootStrapQuery = TableQuery.CombineFilters(localAvailabilitySetQuery, TableOperators.Or, bootStrapQuery);
}
}
var virtualMachineTableQuery = new TableQuery <VirtualMachineCrawlerResponse>().Where(bootStrapQuery);
var crawledVirtualMachineResults = StorageAccountProvider.GetEntities(virtualMachineTableQuery, StorageTableNames.VirtualMachineCrawlerTableName);
crawledVirtualMachineResults = crawledVirtualMachineResults.Where(x => PowerState.Parse(x.State) == PowerState.Running);
foreach (var eachVirtualMachine in crawledVirtualMachineResults)
{
string entryIntoPossibleAvailabilitySetDomainCombinationVmCount;
if (_azureClient.AzureSettings.Chaos.AvailabilitySetChaos.FaultDomainEnabled)
{
entryIntoPossibleAvailabilitySetDomainCombinationVmCount = eachVirtualMachine.AvailabilitySetId + Delimeters.Exclamatory.ToString() + eachVirtualMachine.FaultDomain;
}
else
{
entryIntoPossibleAvailabilitySetDomainCombinationVmCount = eachVirtualMachine.AvailabilitySetId + Delimeters.At.ToString() + eachVirtualMachine.UpdateDomain;
}
if (possibleAvailabilitySetDomainCombinationVmCount.ContainsKey(entryIntoPossibleAvailabilitySetDomainCombinationVmCount))
{
possibleAvailabilitySetDomainCombinationVmCount[entryIntoPossibleAvailabilitySetDomainCombinationVmCount] += 1;
}
else
{
possibleAvailabilitySetDomainCombinationVmCount[entryIntoPossibleAvailabilitySetDomainCombinationVmCount] = 1;
}
}
var possibleAvailableSets = new List <string>(possibleAvailabilitySetDomainCombinationVmCount.Keys);
return(possibleAvailableSets);
}
public void Initialize(ExtensionConfigContext context, StorageAccountProvider storageAccountProvider, IContextGetter <IMessageEnqueuedWatcher> contextGetter)
{
_accountProvider = storageAccountProvider;
_messageEnqueuedWatcherGetter = contextGetter;
// TODO: FACAVAL replace this with queue options. This should no longer be needed.
//context.ApplyConfig(context.Config.Queues, "queues");
// IStorageQueueMessage is the core testing interface
var binding = context.AddBindingRule <QueueAttribute>();
binding
.AddConverter <byte[], CloudQueueMessage>(ConvertByteArrayToCloudQueueMessage)
.AddConverter <string, CloudQueueMessage>(ConvertStringToCloudQueueMessage)
.AddOpenConverter <OpenType.Poco, CloudQueueMessage>(ConvertPocoToCloudQueueMessage);
context // global converters, apply to multiple attributes.
.AddConverter <CloudQueueMessage, byte[]>(ConvertCloudQueueMessageToByteArray)
.AddConverter <CloudQueueMessage, string>(ConvertCloudQueueMessageToString);
var builder = new QueueBuilder(this);
binding.AddValidator(ValidateQueueAttribute);
binding.SetPostResolveHook(ToWriteParameterDescriptorForCollector)
.BindToCollector <CloudQueueMessage>(this);
binding.SetPostResolveHook(ToReadWriteParameterDescriptorForCollector)
.BindToInput <CloudQueue>(builder);
binding.SetPostResolveHook(ToReadWriteParameterDescriptorForCollector)
.BindToInput <CloudQueue>(builder);
}
public IHost ConfigureHost(LogLevel logLevel)
{
_resolver = new RandomNameResolver();
IHost host = new HostBuilder()
.ConfigureDefaultTestHost <HttpDependencyCollectionTests>(b =>
{
b.AddAzureStorage();
})
.ConfigureServices(services =>
{
services.AddSingleton <INameResolver>(_resolver);
services.Configure <FunctionResultAggregatorOptions>(o =>
{
o.IsEnabled = false;
});
})
.ConfigureLogging(b =>
{
b.SetMinimumLevel(logLevel);
b.AddApplicationInsightsWebJobs(o => o.InstrumentationKey = _mockApplicationInsightsKey);
})
.Build();
TelemetryConfiguration telemteryConfiguration = host.Services.GetService <TelemetryConfiguration>();
telemteryConfiguration.TelemetryChannel = _channel;
StorageAccountProvider provider = host.Services.GetService <StorageAccountProvider>();
CloudStorageAccount storageAccount = provider.GetHost().SdkObject;
_blobClient = storageAccount.CreateCloudBlobClient();
_queueClient = storageAccount.CreateCloudQueueClient();
_inputContainerName = _resolver.ResolveInString(InputContainerNamePattern);
_outputContainerName = _resolver.ResolveInString(OutputContainerNamePattern);
_triggerContainerName = _resolver.ResolveInString(TriggerContainerNamePattern);
_outputQueueName = _resolver.ResolveInString(OutputQueueNamePattern);
_triggerQueueName = _resolver.ResolveInString(TriggerQueueNamePattern);
CloudBlobContainer inContainer = _blobClient.GetContainerReference(_inputContainerName);
CloudBlobContainer outContainer = _blobClient.GetContainerReference(_outputContainerName);
CloudBlobContainer triggerContainer = _blobClient.GetContainerReference(_triggerContainerName);
CloudQueue outputQueue = _queueClient.GetQueueReference(_outputQueueName);
_triggerQueue = _queueClient.GetQueueReference(_triggerQueueName);
inContainer.CreateIfNotExistsAsync().Wait();
outContainer.CreateIfNotExistsAsync().Wait();
triggerContainer.CreateIfNotExistsAsync().Wait();
outputQueue.CreateIfNotExistsAsync().Wait();
_triggerQueue.CreateIfNotExistsAsync().Wait();
CloudBlockBlob inBlob = inContainer.GetBlockBlobReference("in");
inBlob.UploadTextAsync("TestData").Wait();
return(host);
}
public NuGetCatalogTriggerAttributeBindingProvider(
INameResolver nameResolver,
StorageAccountProvider accountProvider,
ILoggerFactory loggerFactory)
{
_nameResolver = nameResolver;
_accountProvider = accountProvider;
_loggerFactory = loggerFactory;
}
public static List <T> QueryByRowKey <T>(string rowKey, string tableName) where T : ITableEntity, new()
{
var tableQuery = new TableQuery <T>();
tableQuery = tableQuery.Where(TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, rowKey));
var resultsSet = StorageAccountProvider.GetEntities(tableQuery, tableName);
return(resultsSet.ToList());
}
public void CreateRule(TraceWriter log)
{
try
{
var loadBalancer = GetRandomLoadBalancer();
if (loadBalancer == null)
{
log.Info("Loadbalancer RuleEngine: No load balancer found with virtual machines.");
return;
}
var filteredVmSet = GetVirtualMachineSet(loadBalancer.Id);
if (filteredVmSet == null)
{
log.Info("Loadbalancer RuleEngine: No virtual machines found for the load balancer name: " + loadBalancer.ResourceName);
return;
}
var table = StorageAccountProvider.CreateOrGetTable(StorageTableNames.ScheduledRulesTableName);
if (table == null)
{
return;
}
var count = VmCount(filteredVmSet.Count);
var tasks = new List <Task>();
// do
// {
var randomSets = filteredVmSet.Take(count).ToList();
filteredVmSet = filteredVmSet.Except(randomSets).ToList();
for (var i = 0;
i < randomSets.Count;
i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = randomSets.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
var batchOperation = VirtualMachineHelper.CreateScheduleEntity(batchItems,
_azureClient.AzureSettings.Chaos.SchedulerFrequency,
_azureClient.AzureSettings.Chaos.AzureFaultInjectionActions,
VirtualMachineGroup.LoadBalancer);
var operation = batchOperation;
tasks.Add(table.ExecuteBatchAsync(operation));
}
// } while (filteredVmSet.Any());
Task.WhenAll(tasks);
log.Info("Loadbalancer RuleEngine: Completed creating rule engine.");
}
catch (Exception ex)
{
log.Error("LoadBalancer RuleEngine: Exception thrown. ", ex);
}
}
private void InsertVirtualMachineAvailabilitySetDomainResults(string availabilitySetId, int domainNumber)
{
var virtualMachineQuery = TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("AvailabilitySetId",
QueryComparisons.Equal,
availabilitySetId),
TableOperators.And,
_azureClient.AzureSettings.Chaos.AvailabilitySetChaos.FaultDomainEnabled
? TableQuery.GenerateFilterConditionForInt("FaultDomain",
QueryComparisons.Equal,
domainNumber)
: TableQuery.GenerateFilterConditionForInt("UpdateDomain",
QueryComparisons.Equal,
domainNumber));
//TableQuery.GenerateFilterConditionForInt("AvailabilityZone", QueryComparisons.GreaterThanOrEqual, 0);
var virtualMachinesTableQuery = new TableQuery <VirtualMachineCrawlerResponse>().Where(virtualMachineQuery);
var crawledVirtualMachinesResults = StorageAccountProvider.GetEntities(virtualMachinesTableQuery,
StorageTableNames.VirtualMachineCrawlerTableName);
var virtualMachinesResults = crawledVirtualMachinesResults.ToList();
if (!virtualMachinesResults.Any())
{
return;
}
var domainFlag = !_azureClient.AzureSettings.Chaos.AvailabilitySetChaos.UpdateDomainEnabled;
var batchTasks = new List <Task>();
var table = StorageAccountProvider.CreateOrGetTable(StorageTableNames.ScheduledRulesTableName);
if (table == null)
{
return;
}
for (var i = 0; i < virtualMachinesResults.Count; i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = virtualMachinesResults.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
var scheduledRulesbatchOperation =
VirtualMachineHelper.CreateScheduleEntityForAvailabilitySet(batchItems,
_azureClient.AzureSettings.Chaos.SchedulerFrequency,
_azureClient.AzureSettings.Chaos.AzureFaultInjectionActions,
domainFlag);
if (scheduledRulesbatchOperation.Count <= 0)
{
return;
}
batchTasks.Add(table.ExecuteBatchAsync(scheduledRulesbatchOperation));
}
if (batchTasks.Count > 0)
{
Task.WhenAll(batchTasks);
}
}
public TableAttributeBindingProvider(INameResolver nameResolver, StorageAccountProvider accountProvider)
{
_nameResolver = nameResolver;
_accountProvider = accountProvider ?? throw new ArgumentNullException(nameof(accountProvider));
_entityBindingProvider =
new CompositeEntityArgumentBindingProvider(
new TableEntityArgumentBindingProvider(),
new PocoEntityArgumentBindingProvider()); // Supports all types; must come after other providers
}
public BlobsExtensionConfigProvider(StorageAccountProvider accountProvider,
BlobTriggerAttributeBindingProvider triggerBinder,
IContextGetter <IBlobWrittenWatcher> contextAccessor,
INameResolver nameResolver,
IConverterManager converterManager)
{
_accountProvider = accountProvider;
_triggerBinder = triggerBinder;
_blobWrittenWatcherGetter = contextAccessor;
_nameResolver = nameResolver;
_converterManager = converterManager;
}
/// <summary>Get the list of the availability sets by resource group.
/// And get the virtual machine by resource group and the availability sets.
/// And get the batch operation for the availability sets</summary>
/// <param name="resourceGroupList"></param>
/// <param name="virtualMachinesConcurrent"></param>
/// <param name="batchTasks"></param>
/// <param name="azureClient"></param>
/// <param name="log"></param>
private static void SetTheVirtualMachinesAndAvailabilitySetBatchTask(IEnumerable <IResourceGroup> resourceGroupList,
ConcurrentBag <IEnumerable <IGrouping <string, IVirtualMachine> > > virtualMachinesConcurrent,
ConcurrentBag <Task> batchTasks,
AzureClient azureClient,
TraceWriter log)
{
var availabilitySetCloudTable = StorageAccountProvider.CreateOrGetTable(StorageTableNames.AvailabilitySetCrawlerTableName);
if (availabilitySetCloudTable == null)
{
return;
}
Parallel.ForEach(resourceGroupList, eachResourceGroup =>
{
try
{
var availabilitySetIds = new List <string>();
var availabilitySetsByResourceGroup = azureClient.AzureInstance.AvailabilitySets.ListByResourceGroup(eachResourceGroup.Name);
// table batch operation currently allows only 100 per batch, So ensuring the one batch operation will have only 100 items
var setsByResourceGroup = availabilitySetsByResourceGroup.ToList();
for (var i = 0; i < setsByResourceGroup.Count; i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = setsByResourceGroup.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
// get the availability sets by resource group
// get the availability sets batch operation and get the list of availability set ids
var availabilitySetbatchOperation =
GetAvailabilitySetBatchOperation(batchItems, availabilitySetIds);
// add the batch operation into task list
if (availabilitySetbatchOperation.Count > 0 && availabilitySetCloudTable != null)
{
batchTasks.Add(availabilitySetCloudTable.ExecuteBatchAsync(availabilitySetbatchOperation));
}
}
// Get the virtual machines by resource group and by availability set ids
var virtualMachinesByAvailabilitySetId = GetVirtualMachineListByResourceGroup(eachResourceGroup.Name, availabilitySetIds, azureClient);
if (virtualMachinesByAvailabilitySetId != null && virtualMachinesByAvailabilitySetId.Count > 0)
{
virtualMachinesConcurrent.Add(virtualMachinesByAvailabilitySetId);
}
}
catch (Exception e)
{
log.Error($"timercrawlerforavailableset threw the exception ", e,
"for resource group: " + eachResourceGroup.Name);
}
});
}
protected override async Task OnExecuteAsync(CommandLineApplication app)
{
if (string.IsNullOrEmpty(SubscriptionId) || string.IsNullOrEmpty(ResourceName))
{
ReportError(new ArgumentException("Missing subscriptionId or resource name"));
}
var keyvaultAddress = $"https://{KeyVaultName}.vault.azure.net/";
var azureHelper = new AzureHelper(TenantId);
var strAccountProvider = new StorageAccountProvider(azureHelper, keyvaultAddress);
var storageAccount = await strAccountProvider.GetRPTableReader(SignalRConstants.SignalRInstanceStorageAccount);
var signalrInstanceTbl = new SignalRInstanceTable(SignalRConstants.SignalRInstanceTableName, storageAccount.CreateCloudTableClient());
await Helper.SearchEntity(signalrInstanceTbl, SubscriptionId, ResourceName);
}
public StorageLoadBalancerQueue(
StorageAccountProvider storageAccountProvider,
IOptions <QueuesOptions> queueOptions,
IWebJobsExceptionHandler exceptionHandler,
SharedQueueWatcher sharedWatcher,
ILoggerFactory loggerFactory,
IQueueProcessorFactory queueProcessorFactory)
{
_storageAccountProvider = storageAccountProvider;
_queueOptions = queueOptions.Value;
_exceptionHandler = exceptionHandler;
_sharedWatcher = sharedWatcher;
_loggerFactory = loggerFactory;
_queueProcessorFactory = queueProcessorFactory;
}
/// <summary>Create the virtual machine rules</summary>
/// <param name="log"></param>
public void CreateRule(TraceWriter log)
{
try
{
log.Info("VirtualMachine RuleEngine: Started the creating rules for the virtual machines.");
var vmSets = GetRandomVmSet();
if (vmSets == null)
{
log.Info("VirtualMachine RuleEngine: No virtual machines found..");
return;
}
var table = StorageAccountProvider.CreateOrGetTable(StorageTableNames.ScheduledRulesTableName);
var count = VmCount(vmSets.Count);
var tasks = new List <Task>();
//do
//{
var randomSets = vmSets.Take(count).ToList();
vmSets = vmSets.Except(randomSets).ToList();
for (var i = 0;
i < randomSets.Count;
i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = randomSets.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
var batchOperation = VirtualMachineHelper.CreateScheduleEntity(batchItems,
azureClient.AzureSettings.Chaos.SchedulerFrequency,
azureClient.AzureSettings.Chaos.AzureFaultInjectionActions,
VirtualMachineGroup.VirtualMachines);
if (batchOperation == null)
{
continue;
}
tasks.Add(table.ExecuteBatchAsync(batchOperation));
}
// } while (vmSets.Any());
Task.WhenAll(tasks);
log.Info("VirtualMachine RuleEngine: Completed creating rule engine..");
}
catch (Exception ex)
{
log.Error("VirtualMachine RuleEngine: Exception thrown. ", ex);
}
}
/// <summary>1. Iterate the resource groups to get the scale sets for individual resource group.
/// 2. Convert the List of scale sets into scale set entity and add them into the table batch operation.
/// 3. Get the list of virtual machine instances, convert into entity and them into the table batach operation
/// 3. Execute all the task parallely</summary>
/// <param name="resourceGroups">List of resource groups for the particular subscription.</param>
/// <param name="log">Trace writer instance</param>
private static async Task GetLoadBalancersForResourceGroupsAsync(IEnumerable <IResourceGroup> resourceGroups,
TraceWriter log)
{
try
{
var virtualMachineCloudTable = StorageAccountProvider.CreateOrGetTable(StorageTableNames.VirtualMachineCrawlerTableName);
var loadBalancerTable = StorageAccountProvider.CreateOrGetTable(StorageTableNames.LoadBalancerCrawlerTableName);
if (virtualMachineCloudTable == null || loadBalancerTable == null)
{
return;
}
var batchTasks = new ConcurrentBag <Task>();
var azureClient = new AzureClient();
// using parallel here to run all the resource groups parallelly, parallel is 10times faster than normal foreach.
foreach (var eachResourceGroup in resourceGroups)
//Parallel.ForEach(resourceGroups, eachResourceGroup =>
{
try
{
var loadBalancersList = azureClient.AzureInstance.LoadBalancers
.ListByResourceGroup(eachResourceGroup.Name);
//var count = loadBalancersList.Count();
if (loadBalancersList.Count() > 0)
{
GetVirtualMachineAndLoadBalancerBatch(loadBalancersList.ToList(), batchTasks,
virtualMachineCloudTable,
loadBalancerTable, azureClient, log);
}
}
catch (Exception e)
{
// catch the error, to continue adding other entities to table
log.Error($"timercrawlerforloadbalancer threw the exception ", e,
"GetLoadBalancerForResourceGroups: for the resource group " + eachResourceGroup.Name);
}
}
//);
// execute all batch operation as parallel
await Task.WhenAll(batchTasks);
}
catch (Exception ex)
{
log.Error($"timercrawlerforloadbalancer threw the exception ", ex, "GetLoadBalancerForResourceGroups");
}
}
private IEnumerable <string> GetRecentlyExecutedAvailabilityZoneRegionCombination()
{
var recentlyExecutedAvailabilityZoneRegionCombination = new List <string>();
var possibleAvailabilityZoneRegionCombinationVmCount = new Dictionary <string, int>();
var meanTimeQuery = TableQuery.GenerateFilterConditionForDate("ScheduledExecutionTime",
QueryComparisons.GreaterThanOrEqual,
DateTimeOffset.UtcNow.AddMinutes(-azureClient.AzureSettings.Chaos.MeanTime));
var recentlyExecutedAvailabilityZoneRegionCombinationQuery = TableQuery.GenerateFilterCondition(
"ResourceType",
QueryComparisons.Equal,
VirtualMachineGroup.AvailabilityZones.ToString());
var recentlyExecutedFinalAvailabilityZoneRegionQuery = TableQuery.CombineFilters(meanTimeQuery,
TableOperators.And,
recentlyExecutedAvailabilityZoneRegionCombinationQuery);
var scheduledQuery = new TableQuery <ScheduledRules>().Where(recentlyExecutedFinalAvailabilityZoneRegionQuery);
//Will get the executed query results.
var executedAvilabilityZoneCombinationResults = StorageAccountProvider.GetEntities(scheduledQuery,
StorageTableNames.ScheduledRulesTableName);
if (executedAvilabilityZoneCombinationResults == null)
{
return(recentlyExecutedAvailabilityZoneRegionCombination);
}
foreach (var eachExecutedAvilabilityZoneCombinationResults in executedAvilabilityZoneCombinationResults)
{
if (!eachExecutedAvilabilityZoneCombinationResults.CombinationKey.Contains("!"))
{
continue;
}
if (possibleAvailabilityZoneRegionCombinationVmCount.ContainsKey(eachExecutedAvilabilityZoneCombinationResults.CombinationKey))
{
possibleAvailabilityZoneRegionCombinationVmCount[eachExecutedAvilabilityZoneCombinationResults.CombinationKey] += 1;
}
else
{
possibleAvailabilityZoneRegionCombinationVmCount[eachExecutedAvilabilityZoneCombinationResults.CombinationKey] = 1;
}
}
recentlyExecutedAvailabilityZoneRegionCombination = new List <string>(possibleAvailabilityZoneRegionCombinationVmCount.Keys);
return(recentlyExecutedAvailabilityZoneRegionCombination);
}
public QueueTriggerAttributeBindingProvider(INameResolver nameResolver,
StorageAccountProvider accountProvider,
IOptions <QueuesOptions> queueOptions,
IWebJobsExceptionHandler exceptionHandler,
SharedQueueWatcher messageEnqueuedWatcherSetter,
ILoggerFactory loggerFactory,
IQueueProcessorFactory queueProcessorFactory)
{
_accountProvider = accountProvider ?? throw new ArgumentNullException(nameof(accountProvider));
_queueOptions = (queueOptions ?? throw new ArgumentNullException(nameof(queueOptions))).Value;
_exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler));
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter));
_nameResolver = nameResolver;
_loggerFactory = loggerFactory;
_queueProcessorFactory = queueProcessorFactory;
}
/// <summary>1. Iterate the resource groups to get the virtual machines for individual resource group.
/// 2. Convert the List of virtual machines into entities and add them into tasklist, repeat this process for the VM's under the resource group.
/// 3. Execute all the task parallely</summary>
/// <param name="resourceGroups">List of resource groups for the particular subscription.</param>
/// <param name="log">Trace writer instance</param>
private static async Task GetVirtualMachinesByResourceGroups(IEnumerable <IResourceGroup> resourceGroups,
TraceWriter log)
{
try
{
var virtualMachineCloudTable = StorageAccountProvider.CreateOrGetTable(StorageTableNames.VirtualMachineCrawlerTableName);
if (virtualMachineCloudTable == null)
{
return;
}
var batchTasks = new ConcurrentBag <Task>();
// using parallel here to run all the resource groups parallelly, parallel is 10times faster than normal foreach.
Parallel.ForEach(resourceGroups, async eachResourceGroup =>
{
var virtualMachinesByResourceGroup = await GetVirtualMachinesByResourceGroup(eachResourceGroup, log);
if (virtualMachinesByResourceGroup == null)
{
return;
}
var virtualMachineList = virtualMachinesByResourceGroup.ToList();
// table batch operation currently allows only 100 per batch, So ensuring the one batch operation will have only 100 items
for (var i = 0; i < virtualMachineList.Count(); i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = virtualMachineList.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
var batchOperation = InsertOrReplaceEntitiesIntoTable(batchItems,
eachResourceGroup.Name, log);
if (batchOperation != null)
{
batchTasks.Add(virtualMachineCloudTable.ExecuteBatchAsync(batchOperation));
}
}
});
await Task.WhenAll(batchTasks);
}
catch (Exception e)
{
log.Error($"timercrawlerforvirtualmachines:threw exception", e, "GetVirtualMachinesForResourceGroups");
}
}
public static List <T> QueryByFromToDateForActivities <T>(DateTimeOffset fromDate,
DateTimeOffset toDate,
string propertyName,
string tableName)
where T : ITableEntity, new()
{
var tableQuery = new TableQuery <T>();
var dateFilter = TableQuery.CombineFilters(
TableQuery.GenerateFilterConditionForDate(propertyName, QueryComparisons.GreaterThanOrEqual,
fromDate),
TableOperators.And,
TableQuery.GenerateFilterConditionForDate(propertyName, QueryComparisons.LessThanOrEqual,
toDate));
tableQuery = tableQuery.Where(dateFilter);
var resultsSet = StorageAccountProvider.GetEntities(tableQuery, tableName);
return(resultsSet?.ToList());
}
public static List <VirtualMachineGroup> GetEnabledChaosSet(AzureSettings azureSettings)
{
var enabledChaos = Mappings.GetEnabledChaos(azureSettings);
var selectionQuery = TableQuery.GenerateFilterConditionForDate("ScheduledExecutionTime", QueryComparisons.GreaterThanOrEqual,
DateTimeOffset.UtcNow.AddMinutes(-azureSettings.Chaos.MeanTime));
var scheduledQuery = new TableQuery <ScheduledRules>().Where(selectionQuery);
var executedResults = StorageAccountProvider.GetEntities(scheduledQuery, StorageTableNames.ScheduledRulesTableName);
if (executedResults == null)
{
var chaos = enabledChaos.Where(x => x.Value);
return(chaos.Select(x => x.Key).ToList());
}
var scheduledRuleses = executedResults.ToList();
var executedChaos = scheduledRuleses.Select(x => x.PartitionKey).Distinct().ToList();
var excludedChaos = enabledChaos.Where(x => x.Value && !executedChaos.Contains(x.Key.ToString(), StringComparer.OrdinalIgnoreCase));
return(excludedChaos.Select(x => x.Key).ToList());
}
private static async Task InsertOrReplaceResourceGroupsAsync(List <IResourceGroup> resourceGroups, TraceWriter log)
{
var tableBatchOperation = new TableBatchOperation();
// table batch operation currently allows only 100 per batch, So ensuring the one batch operation will have only 100 items
for (var i = 0; i < resourceGroups.Count; i += TableConstants.TableServiceBatchMaximumOperations)
{
var batchItems = resourceGroups.Skip(i)
.Take(TableConstants.TableServiceBatchMaximumOperations).ToList();
foreach (var eachResourceGroup in batchItems)
{
var resourceGroupCrawlerResponseEntity =
new ResourceGroupCrawlerResponse("", eachResourceGroup.Name);
try
{
resourceGroupCrawlerResponseEntity.Id = eachResourceGroup.Id;
resourceGroupCrawlerResponseEntity.RegionName = eachResourceGroup.RegionName;
tableBatchOperation.InsertOrReplace(resourceGroupCrawlerResponseEntity);
}
catch (Exception ex)
{
resourceGroupCrawlerResponseEntity.Error = ex.Message;
log.Error($"timercrawlerresourcegroups threw exception ", ex, "timercrawlerresourcegroups");
}
}
}
if (tableBatchOperation.Count > 0)
{
try
{
var table = StorageAccountProvider.CreateOrGetTable(StorageTableNames.ResourceGroupCrawlerTableName);
await table.ExecuteBatchAsync(tableBatchOperation);
}
catch (Exception ex)
{
log.Error($"timercrawlerresourcegroups threw exception ", ex, "timercrawlerresourcegroups");
}
}
}
/// <summary>Get the scheduled rules for the chaos execution.</summary>
/// <returns></returns>
private static IEnumerable <ScheduledRules> GetScheduledRulesForExecution(TraceWriter log)
{
try
{
var azureSettings = new AzureClient().AzureSettings;
var dateFilterByUtc = TableQuery.GenerateFilterConditionForDate("ScheduledExecutionTime", QueryComparisons.GreaterThanOrEqual,
DateTimeOffset.UtcNow);
var dateFilterByFrequency = TableQuery.GenerateFilterConditionForDate("ScheduledExecutionTime", QueryComparisons.LessThanOrEqual,
DateTimeOffset.UtcNow.AddMinutes(TriggerFrequency));
var filter = TableQuery.CombineFilters(dateFilterByUtc, TableOperators.And, dateFilterByFrequency);
var scheduledQuery = new TableQuery <ScheduledRules>().Where(filter);
return(StorageAccountProvider.GetEntities(scheduledQuery, StorageTableNames.ScheduledRulesTableName));
}
catch (Exception e)
{
log.Error($"TimerTrigger function threw exception", e, "GetScheduledRulesForExecution");
throw;
}
}
public static List <ScheduledRules> QuerySchedulesByMeanTime <T>(AzureSettings azureSettings,
string tableName,
string filter = "")
{
var tableQuery = new TableQuery <ScheduledRules>();
var dateFilter = TableQuery.CombineFilters(TableQuery.GenerateFilterConditionForDate("ScheduledExecutionTime",
QueryComparisons.LessThanOrEqual,
DateTimeOffset.UtcNow),
TableOperators.And,
TableQuery.GenerateFilterConditionForDate("ScheduledExecutionTime",
QueryComparisons.GreaterThanOrEqual,
DateTimeOffset.UtcNow.AddMinutes(-azureSettings.Chaos.MeanTime)));
var combineFilter = !string.IsNullOrWhiteSpace(filter)
? TableQuery.CombineFilters(dateFilter,
TableOperators.And,
filter)
: dateFilter;
tableQuery = tableQuery.Where(combineFilter);
var resultsSet = StorageAccountProvider.GetEntities(tableQuery, tableName);
return(resultsSet.ToList());
}
