// ARM private async Task LoadAzureResources() { LoadMonitoringToolResources(); var subscriptions = await CsmManager.GetSubscriptions(); HostingEnvironment.QueueBackgroundWorkItem(_ => { foreach (var subscription in subscriptions) { _backgroundQueueManager.LoadSubscription(subscription); } }); }
internal async Task CleanupSubscriptions() { try { _cleanupOperationsTriggered++; var subscriptions = await CsmManager.GetSubscriptions(); var totalDeletedRGs = 0; Stopwatch sw = new Stopwatch(); sw.Start(); foreach (var sub in subscriptions) { var s = await new Subscription(sub).Load(false); SimpleTrace.Diagnostics.Information($"Deleting resources in {s.Type} subscription {s.SubscriptionId}"); //Delete any leaking resourcegroups in subscription that cannot be loaded var csmResourceGroups = await s.LoadResourceGroupsForSubscription(); var deleteLeakingRGs = csmResourceGroups.value .Where(p => !LoadedResourceGroups.Any(p2 => string.Equals(p.id, p2.CsmId, StringComparison.OrdinalIgnoreCase))).GroupBy(rg => rg.location) .Select(g => new { Region = g.Key, ResourceGroups = g.Select(r => r), Count = g.Count() }) .Where(g => g.Count > s.ResourceGroupsPerGeoRegion) .Select(g => g.ResourceGroups.Where(rg => rg.tags != null && !rg.tags.ContainsKey("UserId"))) .SelectMany(i => i); totalDeletedRGs += (deleteLeakingRGs == null)? 0: deleteLeakingRGs.Count(); AppInsights.TelemetryClient.TrackMetric("deletedLeakingRGs", deleteLeakingRGs.Count()); Parallel.ForEach(deleteLeakingRGs, async resourceGroup => { try { var georegion = CsmManager.RegionHashTable[resourceGroup.location].ToString(); SimpleTrace.Diagnostics.Information($"Deleting leaked {georegion} resource {resourceGroup.name}"); await new ResourceGroup(s.SubscriptionId, resourceGroup.name, georegion).Delete(false); } catch (Exception ex) { SimpleTrace.Diagnostics.Error($"Leaking RG Delete Exception:{ex.ToString()}-{ex.StackTrace}-{ex.InnerException?.StackTrace.ToString() ?? String.Empty}"); } }); } AppInsights.TelemetryClient.TrackMetric("totalDeletedLeakingRGs", totalDeletedRGs); AppInsights.TelemetryClient.TrackMetric("leakingRGsCleanupTime", sw.Elapsed.TotalSeconds); //Delete any duplicate resourcegroups in same subscription loaded in the same region //or create any missing resourcegroups in a region IList <Tuple <string, MakeSubscriptionFreeTrialResult> > subscriptionStates = new List <Tuple <string, MakeSubscriptionFreeTrialResult> >(); var deletedDuplicateRGs = 0; var createdMissingRGs = 0; foreach (var subscription in subscriptions) { var sub = new Subscription(subscription); sub.ResourceGroups = LoadedResourceGroups.Where(r => r.SubscriptionId == sub.SubscriptionId); var trialsubresult = sub.MakeTrialSubscription(); if (trialsubresult.ToCreateInRegions.Any() || trialsubresult.ToDelete.Any()) { subscriptionStates.Add(new Tuple <string, MakeSubscriptionFreeTrialResult>(subscription, trialsubresult)); } } AppInsights.TelemetryClient.TrackMetric("subsWithIncorrectRGs", subscriptionStates.Count()); foreach (var subscriptionState in subscriptionStates) { foreach (var geoRegion in subscriptionState.Item2.ToCreateInRegions) { CreateResourceGroupOperation(subscriptionState.Item1, geoRegion); createdMissingRGs++; } foreach (var resourceGroup in subscriptionState.Item2.ToDelete) { RemoveFromFreeQueue(resourceGroup); DeleteResourceGroupOperation(resourceGroup); deletedDuplicateRGs++; } } AppInsights.TelemetryClient.TrackMetric("createdMissingRGs", createdMissingRGs); AppInsights.TelemetryClient.TrackMetric("deletedDuplicateRGs", deletedDuplicateRGs); AppInsights.TelemetryClient.TrackMetric("fullCleanupTime", sw.Elapsed.TotalSeconds); sw.Stop(); } catch (Exception e) { SimpleTrace.Diagnostics.Fatal(e, "CleanupSubscriptions error, Count {Count}", Interlocked.Increment(ref _maintainResourceGroupListErrorCount)); } }
internal async Task CleanupSubscriptions() { try { _cleanupOperationsTriggered++; var subscriptions = await CsmManager.GetSubscriptions(); Stopwatch sw = new Stopwatch(); sw.Start(); //Delete any duplicate resourcegroups in same subscription loaded in the same region //or create any missing resourcegroups in a region List <ResourceGroup> toDelete = new List <ResourceGroup>(); List <ResourceGroup> ready = new List <ResourceGroup>(); var deletedDuplicateRGs = 0; var createdMissingRGs = 0; var createdMissingTemplates = 0; foreach (var subscription in subscriptions) { var sub = new Subscription(subscription); await sub.Load(); //sub.ResourceGroups = (await LoadedResourceGroups()).Where(r => r.SubscriptionId == sub.SubscriptionId); var trialsubresult = sub.GetSubscriptionStats(); toDelete.AddRange(trialsubresult.ToDelete); ready.AddRange(trialsubresult.Ready); } var rand = new Random(); foreach (var resourceGroup in toDelete) { //RemoveFromFreeResourcesQueue(resourceGroup); DeleteResourceGroupOperation(resourceGroup); deletedDuplicateRGs++; } foreach (var template in TemplatesManager.GetTemplates().Where(a => a.QueueSizeToMaintain > 0)) { var deployedTemplates = ready.Count(a => a.DeployedTemplateName == template.Name); var delta = template.QueueSizeToMaintain - deployedTemplates; SimpleTrace.TraceInformation($"Template {template.Name} has {deployedTemplates} RGs deployed and requires {template.QueueSizeToMaintain}"); for (int i = 1; i <= delta; i++) { SimpleTrace.TraceInformation($"Template {template.Name} creating {i} of {delta}"); CreateResourceGroupOperation(template.Config.Subscriptions.OrderBy(a => Guid.NewGuid()).First(), template.Config.Regions.OrderBy(a => Guid.NewGuid()).First(), template.Name); createdMissingTemplates++; } for (int i = 1; i <= -delta; i++) { var resourceGroup = await StorageHelper.GetQueueMessage(template.QueueName); if (resourceGroup != null) { SimpleTrace.TraceInformation($"Template {template.Name} deleting {i} of {-delta}->{resourceGroup.CsmId}"); //RemoveFromFreeResourcesQueue(resourceGroup); DeleteResourceGroupOperation(resourceGroup); } } } AppInsights.TelemetryClient.TrackMetric("createdMissingTemplates", createdMissingTemplates); AppInsights.TelemetryClient.TrackMetric("createdMissingRGs", createdMissingRGs); AppInsights.TelemetryClient.TrackMetric("deletedDuplicateRGs", deletedDuplicateRGs); AppInsights.TelemetryClient.TrackMetric("fullCleanupTime", sw.Elapsed.TotalSeconds); sw.Stop(); } catch (Exception e) { SimpleTrace.Diagnostics.Fatal(e, "CleanupSubscriptions error, Count {Count}", Interlocked.Increment(ref _maintainResourceGroupListErrorCount)); } }