public CollectionCreator(String settingsFile)
#endif
{
try
{
csmManager = new CsmManager();
if (String.IsNullOrEmpty(settingsFile))
{
settingsFile = CCUtils.GetSettingsFilePath();
}
if (File.Exists(settingsFile))
{
config = CCConfiguration.FromXml(settingsFile);
}
if (config == null)
{
String errMsg = String.Format("{0}: [{1}], error code [{2}]", CCConstants.E0100, settingsFile ?? String.Empty, (int)CCEnums.CCErrorCodes.E0100);
throw new Exception(errMsg);
}
}
catch (Exception ex)
{
ILog.LogError(ex);
throw (ex);
}
}
// ARM
private async Task LoadAzureResources()
{
// Load all subscriptions
var csmSubscriptions = await CsmManager.GetSubscriptionNamesToIdMap();
var subscriptions = SimpleSettings.Subscriptions.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
//It can be either a displayName or a subscriptionId
.Select(s => s.Trim())
.Where(n =>
{
Guid temp;
return(csmSubscriptions.ContainsKey(n) || Guid.TryParse(n, out temp));
})
.Select(sn =>
{
Guid temp;
if (Guid.TryParse(sn, out temp))
{
return(sn);
}
else
{
return(csmSubscriptions[sn]);
}
});
HostingEnvironment.QueueBackgroundWorkItem(_ =>
{
foreach (var subscription in subscriptions)
{
_backgroundQueueManager.LoadSubscription(subscription);
}
});
}
private void LoadMonitoringToolResources()
{
var subscription = CsmManager.LoadMonitoringToolsSubscription();
HostingEnvironment.QueueBackgroundWorkItem(_ =>
{
_backgroundQueueManager.LoadMonitoringToolSubscription(subscription);
});
}
private void CreateResourceGroupOperation(string subscriptionId, string geoRegion)
{
AddOperation(new BackgroundOperation <ResourceGroup>
{
Description = $"Creating resourceGroup in {subscriptionId} in {geoRegion}",
Type = OperationType.ResourceGroupCreate,
Task = CsmManager.CreateResourceGroup(subscriptionId, geoRegion),
RetryAction = () => CreateResourceGroupOperation(subscriptionId, geoRegion)
});
}
private void SubscriptionCleanup(Subscription subscription)
{
AddOperation(new BackgroundOperation <Subscription>
{
Description = $"Cleaning subscriptions",
Type = OperationType.SubscriptionCleanup,
Task = CsmManager.SubscriptionCleanup(subscription),
RetryAction = () => SubscriptionCleanup(subscription)
});
}
// ARM
private async Task LoadAzureResources()
{
LoadMonitoringToolResources();
var subscriptions = await CsmManager.GetSubscriptions();
HostingEnvironment.QueueBackgroundWorkItem(_ =>
{
foreach (var subscription in subscriptions)
{
_backgroundQueueManager.LoadSubscription(subscription);
}
});
}
internal async Task CleanupSubscriptions()
{
try
{
_cleanupOperationsTriggered++;
var subscriptions = await CsmManager.GetSubscriptions();
var totalDeletedRGs = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
foreach (var sub in subscriptions)
{
var s = await new Subscription(sub).Load(false);
SimpleTrace.Diagnostics.Information($"Deleting resources in {s.Type} subscription {s.SubscriptionId}");
//Delete any leaking resourcegroups in subscription that cannot be loaded
var csmResourceGroups = await s.LoadResourceGroupsForSubscription();
var deleteLeakingRGs = csmResourceGroups.value
.Where(p => !LoadedResourceGroups.Any(p2 => string.Equals(p.id, p2.CsmId, StringComparison.OrdinalIgnoreCase))).GroupBy(rg => rg.location)
.Select(g => new { Region = g.Key, ResourceGroups = g.Select(r => r), Count = g.Count() })
.Where(g => g.Count > s.ResourceGroupsPerGeoRegion)
.Select(g => g.ResourceGroups.Where(rg => rg.tags != null && !rg.tags.ContainsKey("UserId")))
.SelectMany(i => i);
totalDeletedRGs += (deleteLeakingRGs == null)? 0: deleteLeakingRGs.Count();
AppInsights.TelemetryClient.TrackMetric("deletedLeakingRGs", deleteLeakingRGs.Count());
Parallel.ForEach(deleteLeakingRGs, async resourceGroup =>
{
try
{
var georegion = CsmManager.RegionHashTable[resourceGroup.location].ToString();
SimpleTrace.Diagnostics.Information($"Deleting leaked {georegion} resource {resourceGroup.name}");
await new ResourceGroup(s.SubscriptionId, resourceGroup.name, georegion).Delete(false);
}
catch (Exception ex)
{
SimpleTrace.Diagnostics.Error($"Leaking RG Delete Exception:{ex.ToString()}-{ex.StackTrace}-{ex.InnerException?.StackTrace.ToString() ?? String.Empty}");
}
});
}
AppInsights.TelemetryClient.TrackMetric("totalDeletedLeakingRGs", totalDeletedRGs);
AppInsights.TelemetryClient.TrackMetric("leakingRGsCleanupTime", sw.Elapsed.TotalSeconds);
//Delete any duplicate resourcegroups in same subscription loaded in the same region
//or create any missing resourcegroups in a region
IList <Tuple <string, MakeSubscriptionFreeTrialResult> > subscriptionStates = new List <Tuple <string, MakeSubscriptionFreeTrialResult> >();
var deletedDuplicateRGs = 0;
var createdMissingRGs = 0;
foreach (var subscription in subscriptions)
{
var sub = new Subscription(subscription);
sub.ResourceGroups = LoadedResourceGroups.Where(r => r.SubscriptionId == sub.SubscriptionId);
var trialsubresult = sub.MakeTrialSubscription();
if (trialsubresult.ToCreateInRegions.Any() || trialsubresult.ToDelete.Any())
{
subscriptionStates.Add(new Tuple <string, MakeSubscriptionFreeTrialResult>(subscription, trialsubresult));
}
}
AppInsights.TelemetryClient.TrackMetric("subsWithIncorrectRGs", subscriptionStates.Count());
foreach (var subscriptionState in subscriptionStates)
{
foreach (var geoRegion in subscriptionState.Item2.ToCreateInRegions)
{
CreateResourceGroupOperation(subscriptionState.Item1, geoRegion);
createdMissingRGs++;
}
foreach (var resourceGroup in subscriptionState.Item2.ToDelete)
{
RemoveFromFreeQueue(resourceGroup);
DeleteResourceGroupOperation(resourceGroup);
deletedDuplicateRGs++;
}
}
AppInsights.TelemetryClient.TrackMetric("createdMissingRGs", createdMissingRGs);
AppInsights.TelemetryClient.TrackMetric("deletedDuplicateRGs", deletedDuplicateRGs);
AppInsights.TelemetryClient.TrackMetric("fullCleanupTime", sw.Elapsed.TotalSeconds);
sw.Stop();
}
catch (Exception e)
{
SimpleTrace.Diagnostics.Fatal(e, "CleanupSubscriptions error, Count {Count}", Interlocked.Increment(ref _maintainResourceGroupListErrorCount));
}
}
public static async Task MainAsync()
{
var subscriptionNames = System.Environment.GetEnvironmentVariable("Subscriptions").Split(',');
var csmSubscriptions = await CsmManager.GetSubscriptionNamesToIdMap();
var subscriptionsIds = SimpleSettings.Subscriptions.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
//It can be either a displayName or a subscriptionId
.Select(s => s.Trim())
.Where(n =>
{
Guid temp;
return(csmSubscriptions.ContainsKey(n) || Guid.TryParse(n, out temp));
})
.Select(sn =>
{
Guid temp;
if (Guid.TryParse(sn, out temp))
{
return(sn);
}
else
{
return(csmSubscriptions[sn]);
}
});
//var subscriptionNames = new[] { "bd5cf6af-4d44-449e-adaf-20626ae5013e" };
var startTime = DateTime.UtcNow;
Action <object> console = (s) => System.Console.WriteLine("[" + (DateTime.UtcNow - startTime).TotalMilliseconds + "] " + s);
Action <Subscription> printSub = (sub) => sub.ResourceGroups.ToList().ForEach(e =>
console(string.Format("RG: {0} has {1} sites, named: {2}",
e.ResourceGroupName,
e.Sites.Count(),
e.Sites.Count() > 0
? e.Sites.Select(s => s.SiteName).Aggregate((a, b) => string.Join(",", a, b))
: "No sites")
));
console("start loading subscriptions");
console("We have " + subscriptionsIds.Count() + " subscriptions");
var subscriptions = await subscriptionsIds.Select(s => new Subscription(s).Load()).WhenAll();
console("done loading subscriptions");
console("subscriptions have: " + subscriptions.Aggregate(0, (count, sub) => count += sub.ResourceGroups.Count()) + " resourceGroups");
//foreach (var resourcegroup in subscriptions.SelectMany(subscription => subscription.ResourceGroups))
// {
// try
// {
// if (!resourcegroup.IsSimpleWAWS)
// {
// console($" Replacing Resource Group : {resourcegroup.CsmId}");
// await resourcegroup.DeleteAndCreateReplacement(true);
// console($" Replaced");
// }
// }
// catch (Exception ex)
// {
// console(ex.ToString());
// }
//}
console("calling MakeTrialSubscription on all subscriptions");
await subscriptions.Select(async s =>
{
var result = s.MakeTrialSubscription();
foreach (var resourceGroup in result.Ready)
{
try
{
await resourceGroup.PutInDesiredState();
}
catch (Exception ex)
{
console(
$"RG PIDS Exception:{ex.ToString()}-{ex.StackTrace}-{ex.InnerException?.StackTrace.ToString() ?? String.Empty}");
}
}
foreach (var geoRegion in result.ToCreateInRegions)
{
try
{
await CsmManager.CreateResourceGroup(s.SubscriptionId, geoRegion).Result.PutInDesiredState();
}
catch (Exception ex)
{
console($"GR Create Exception:{ex.ToString()}-{ex.StackTrace}-{ex.InnerException?.StackTrace.ToString() ?? String.Empty}");
}
}
foreach (var resourceGroup in result.ToDelete)
{
try { await resourceGroup.Delete(true); }
catch (Exception ex)
{
console($"RG Delete Exception:{ex.ToString()}-{ex.StackTrace}-{ex.InnerException?.StackTrace.ToString() ?? String.Empty}");
}
}
}).WhenAll();
console("done calling make trial subscription");
////console("subscriptions have: " + subscriptions.Aggregate(0, (count, sub) => count += sub.ResourceGroups.Count()) + " resourceGroups");
////console(subscriptions.Aggregate(0, (count, sub) => count += sub.ResourceGroups.Count()));
////console("make free trial");
//////Parallel.ForEach(
////// subscriptions.SelectMany(subscription => subscription.ResourceGroups), async (resourcegroup) =>
////// {
////// console($"Deleting resourcegroup:{resourcegroup.ResourceGroupName} from \tsubscription:{resourcegroup.SubscriptionId}");
////// await resourcegroup.Delete(true);
////// });
console("Re-start loading subscriptions");
console("We have " + subscriptionsIds.Count() + " subscriptions");
subscriptions = await subscriptionsIds.Select(s => new Subscription(s).Load()).WhenAll();
console("done loading subscriptions");
//Parallel.ForEach(
// subscriptions.SelectMany(subscription => subscription.ResourceGroups), async (resourcegroup) =>
// {
////foreach (var resourcegroup in subscriptions.SelectMany(subscription => subscription.ResourceGroups))
////{
//// try
//// {
//// console($" Replacing Resource Group : {resourcegroup.CsmId}");
//// await resourcegroup.Delete(true);//DeleteAndCreateReplacement(true);
//// console($" Replaced");
//// }
//// catch (Exception ex)
//// {
//// console(ex.ToString());
//// }
////}
//}
//);
//subscriptions.ToList().ForEach(printSub);
console("Done");
}
public static async Task FixFunctionsSetting()
{
var startTime = DateTime.UtcNow;
Action <object> console = (s) => System.Console.WriteLine("[" + (DateTime.UtcNow - startTime).TotalMilliseconds + "] " + s);
console("start");
var csmSubscriptions = await CsmManager.GetSubscriptionNamesToIdMap();
var subscriptionsIds = SimpleSettings.Subscriptions.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
//It can be either a displayName or a subscriptionId
.Select(s => s.Trim())
.Where(n =>
{
Guid temp;
return(csmSubscriptions.ContainsKey(n) || Guid.TryParse(n, out temp));
})
.Select(sn =>
{
Guid temp;
if (Guid.TryParse(sn, out temp))
{
return(sn);
}
else
{
return(csmSubscriptions[sn]);
}
});
console("start loading subscriptions");
console("We have " + subscriptionsIds.Count() + " subscriptions");
var subscriptions = await subscriptionsIds.Select(s => new Subscription(s).Load()).WhenAll();
console("done loading subscriptions");
console("subscriptions have: " + subscriptions.Aggregate(0, (count, sub) => count += sub.ResourceGroups.Count()) + " resourceGroups");
foreach (var resourcegroup in subscriptions.SelectMany(subscription => subscription.ResourceGroups))
{
try
{
console($" Updating Resource Group : {resourcegroup.CsmId}");
await resourcegroup.Load();
var site = resourcegroup.Sites.First(s => s.IsSimpleWAWSOriginalSite);
if (site != null && site.AppSettings.ContainsKey("FUNCTIONS_EXTENSION_VERSION"))
{
site.AppSettings.Remove("FUNCTIONS_EXTENSION_VERSION");
}
await site.UpdateAppSettings();
site = resourcegroup.Sites.First(s => !s.IsSimpleWAWSOriginalSite);
if (site != null && site.AppSettings.ContainsKey("FUNCTIONS_EXTENSION_VERSION") &&
!string.Equals(site.AppSettings["FUNCTIONS_EXTENSION_VERSION"], SimpleSettings.FunctionsExtensionVersion))
{
site.AppSettings["FUNCTIONS_EXTENSION_VERSION"] = SimpleSettings.FunctionsExtensionVersion;
}
await site.UpdateAppSettings();
console($" Updated");
}
catch (Exception ex)
{
console(ex.ToString());
}
}
}
public static async Task MainAsync()
{
var subscriptionNames = System.Environment.GetEnvironmentVariable("Subscriptions").Split(',');
var csmSubscriptions = await CsmManager.GetSubscriptionNamesToIdMap();
var subscriptionsIds = SimpleSettings.Subscriptions.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
//It can be either a displayName or a subscriptionId
.Select(s => s.Trim())
.Where(n =>
{
Guid temp;
return(csmSubscriptions.ContainsKey(n) || Guid.TryParse(n, out temp));
})
.Select(sn =>
{
Guid temp;
if (Guid.TryParse(sn, out temp))
{
return(sn);
}
else
{
return(csmSubscriptions[sn]);
}
});
//var subscriptionNames = new[] { "bd5cf6af-4d44-449e-adaf-20626ae5013e" };
var startTime = DateTime.UtcNow;
Action <object> console = (s) => System.Console.WriteLine("[" + (DateTime.UtcNow - startTime).TotalMilliseconds + "] " + s);
Action <Subscription> printSub = (sub) => sub.ResourceGroups.ToList().ForEach(e =>
console(string.Format("RG: {0} has {1} sites, named: {2}",
e.ResourceGroupName,
e.Sites.Count(),
e.Sites.Count() > 0
? e.Sites.Select(s => s.SiteName).Aggregate((a, b) => string.Join(",", a, b))
: "No sites")
));
console("start loading subscriptions");
console("We have " + subscriptionsIds.Count() + " subscriptions");
var subscriptions = await subscriptionsIds.Select(s => new Subscription(s).Load()).WhenAll();
console("done loading subscriptions");
console("subscriptions have: " + subscriptions.Aggregate(0, (count, sub) => count += sub.ResourceGroups.Count()) + " resourceGroups");
console("subscriptions have: " + subscriptions.Aggregate(0, (count, sub) => count += sub.ResourceGroups.Where(r => !r.Tags.ContainsKey("CommonApiAppsDeployed")).Count()) + " bad resourceGroups");
//console("calling MakeTrialSubscription on all subscriptions");
//subscriptions = await subscriptions.Select(s => s.MakeTrialSubscription()).WhenAll();
//console("done calling make trial subscription");
console("subscriptions have: " + subscriptions.Aggregate(0, (count, sub) => count += sub.ResourceGroups.Count()) + " resourceGroups");
console(subscriptions.Aggregate(0, (count, sub) => count += sub.ResourceGroups.Count()));
console("make free trial");
foreach (var subscription in subscriptions)
{
var freeTrial = subscription.MakeTrialSubscription();
console(subscription.SubscriptionId + "\thas\t" + freeTrial.Ready.Count() + "\twill delete\t" + freeTrial.ToDelete.Count() + "\tcreate\t" + freeTrial.ToCreateInRegions.Count());
}
//await Task.WhenAll(subscriptions.Select(subscription => subscription.ResourceGroups.Where(r => !r.Tags.ContainsKey("CommonApiAppsDeployed")).Select(rg => rg.Delete(false))).SelectMany(i => i));
//subscriptions.ToList().ForEach(printSub);
console("Done");
}
internal async Task CleanupSubscriptions()
{
try
{
_cleanupOperationsTriggered++;
var subscriptions = await CsmManager.GetSubscriptions();
Stopwatch sw = new Stopwatch();
sw.Start();
//Delete any duplicate resourcegroups in same subscription loaded in the same region
//or create any missing resourcegroups in a region
List <ResourceGroup> toDelete = new List <ResourceGroup>();
List <ResourceGroup> ready = new List <ResourceGroup>();
var deletedDuplicateRGs = 0;
var createdMissingRGs = 0;
var createdMissingTemplates = 0;
foreach (var subscription in subscriptions)
{
var sub = new Subscription(subscription);
await sub.Load();
//sub.ResourceGroups = (await LoadedResourceGroups()).Where(r => r.SubscriptionId == sub.SubscriptionId);
var trialsubresult = sub.GetSubscriptionStats();
toDelete.AddRange(trialsubresult.ToDelete);
ready.AddRange(trialsubresult.Ready);
}
var rand = new Random();
foreach (var resourceGroup in toDelete)
{
//RemoveFromFreeResourcesQueue(resourceGroup);
DeleteResourceGroupOperation(resourceGroup);
deletedDuplicateRGs++;
}
foreach (var template in TemplatesManager.GetTemplates().Where(a => a.QueueSizeToMaintain > 0))
{
var deployedTemplates = ready.Count(a => a.DeployedTemplateName == template.Name);
var delta = template.QueueSizeToMaintain - deployedTemplates;
SimpleTrace.TraceInformation($"Template {template.Name} has {deployedTemplates} RGs deployed and requires {template.QueueSizeToMaintain}");
for (int i = 1; i <= delta; i++)
{
SimpleTrace.TraceInformation($"Template {template.Name} creating {i} of {delta}");
CreateResourceGroupOperation(template.Config.Subscriptions.OrderBy(a => Guid.NewGuid()).First(), template.Config.Regions.OrderBy(a => Guid.NewGuid()).First(), template.Name);
createdMissingTemplates++;
}
for (int i = 1; i <= -delta; i++)
{
var resourceGroup = await StorageHelper.GetQueueMessage(template.QueueName);
if (resourceGroup != null)
{
SimpleTrace.TraceInformation($"Template {template.Name} deleting {i} of {-delta}->{resourceGroup.CsmId}");
//RemoveFromFreeResourcesQueue(resourceGroup);
DeleteResourceGroupOperation(resourceGroup);
}
}
}
AppInsights.TelemetryClient.TrackMetric("createdMissingTemplates", createdMissingTemplates);
AppInsights.TelemetryClient.TrackMetric("createdMissingRGs", createdMissingRGs);
AppInsights.TelemetryClient.TrackMetric("deletedDuplicateRGs", deletedDuplicateRGs);
AppInsights.TelemetryClient.TrackMetric("fullCleanupTime", sw.Elapsed.TotalSeconds);
sw.Stop();
}
catch (Exception e)
{
SimpleTrace.Diagnostics.Fatal(e, "CleanupSubscriptions error, Count {Count}", Interlocked.Increment(ref _maintainResourceGroupListErrorCount));
}
}
public static async Task MainAsync()
{
foreach (var a in SimpleSettings.GeoRegions.Split(','))
{
georegionHashMap.Add(a.ToLowerInvariant().Replace(" ", ""), a);
}
//var subscriptionNames = System.Environment.GetEnvironmentVariable("Subscriptions").Split(',');
var csmSubscriptions = await CsmManager.GetSubscriptionNamesToIdMap();
var subscriptionsIds = SimpleSettings.Subscriptions.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
//.Union(SimpleSettings.LinuxSubscriptions.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
//.Union(SimpleSettings.MonitoringToolsSubscription.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
//It can be either a displayName or a subscriptionId
.Select(s => s.Trim())
.Where(n =>
{
Guid temp;
return(csmSubscriptions.ContainsKey(n) || Guid.TryParse(n, out temp));
})
.Select(sn =>
{
Guid temp;
if (Guid.TryParse(sn, out temp))
{
return(sn);
}
else
{
return(csmSubscriptions[sn]);
}
}).OrderByDescending(s => s);
//var subscriptionNames = new[] { "bd5cf6af-4d44-449e-adaf-20626ae5013e" };
var startTime = DateTime.UtcNow;
Action <object> console = (s) => System.Console.WriteLine("[" + (DateTime.UtcNow - startTime).TotalMilliseconds + "] " + s);
Action <Subscription> printSub = (sub) => sub.ResourceGroups.ToList().ForEach(e =>
console(string.Format("RG: {0} has {1} sites, named: {2}",
e.ResourceGroupName,
e.Sites.Count(),
e.Sites.Count() > 0
? e.Sites.Select(s => s.SiteName).Aggregate((a, b) => string.Join(",", a, b))
: "No sites")
));
console("start loading subscriptions");
console("We have " + subscriptionsIds.Count() + " subscriptions");
//var subscriptions = await subscriptionsIds.Select(s => new Subscription(s).Load(false)).WhenAll();
//console("done loading subscriptions");
//console("subscriptions have: " + subscriptions.Aggregate(0, (count, sub) => count += sub.ResourceGroups.Count()) + " resourceGroups");
//var resources = await CsmManager.GetLoadedResources();
console("calling MakeTrialSubscription on all subscriptions");
foreach (var sub in subscriptionsIds)
{
var s = await new Subscription(sub).Load(false);
console($"Deleting resources in {s.Type} subscription {s.SubscriptionId}");
var csmResourceGroups = await s.LoadResourceGroupsForSubscription();
//var deleteExtras = csmResourceGroups.value
// .GroupBy(rg => rg.location)
// .Select(g => new { Region = g.Key, ResourceGroups = g.Select(r => r), Count = g.Count() })
// .Where(g => g.Count > s.ResourceGroupsPerGeoRegion)
// .Select(g => g.ResourceGroups.Where(rg => rg.tags!=null && !rg.tags.ContainsKey("UserId") ).Skip((s.ResourceGroupsPerGeoRegion)))
// .SelectMany(i => i);
Parallel.ForEach(csmResourceGroups.value, async(resourceGroup) =>
{
try
{
var georegion = getGeoRegionfromLocation(resourceGroup.location);
console($"Deleting leaked {georegion} resource {resourceGroup.name}");
await new ResourceGroup(s.SubscriptionId, resourceGroup.name, georegion).Delete(false);
}
catch (Exception ex)
{
console($"Extra RG Delete Exception:{ex.ToString()}-{ex.StackTrace}-{ex.InnerException?.StackTrace.ToString() ?? String.Empty}");
}
});
//var result = s.MakeTrialSubscription();
//Parallel.ForEach(result.ToDelete, async (resourceGroup) =>
//{
// try
// {
// console($"Deleting {resourceGroup.ResourceGroupName} resource in {s.Type} subscription {s.SubscriptionId}");
// await resourceGroup.Delete(false);
// }
// catch (Exception ex)
// {
// console($"RG Delete Exception:{ex.ToString()}-{ex.StackTrace}-{ex.InnerException?.StackTrace.ToString() ?? String.Empty}");
// }
//});
}
//subscriptions.ToList().ForEach(printSub);
console("Done");
}
