/// <summary>
/// If any of the scope in this.rolesToAssign is marked with CURRENT_RESOURCE_GROUP_SCOPE placeholder then
/// resolve it and replace the placeholder with actual resource group scope (id).
/// </summary>
/// <param name="scaleSet">The virtual machine scale set.</param>
/// <return>An observable that emits true once if there was a scope to resolve, otherwise emits false once.</return>
///GENMHASH:1830FCA3FFF54663B3A5702EF2F5721B:F61A8EF35EDD42D392559C585A71073F
private async Task <bool> ResolveCurrentResourceGroupScopeAsync(IVirtualMachineScaleSet scaleSet, CancellationToken cancellationToken = default(CancellationToken))
{
IEnumerable <string> keysWithCurrentResourceGroupScopeForRoles = this.rolesToAssign
.Where(role => role.Value.Item1.Equals(CURRENT_RESOURCE_GROUP_SCOPE, System.StringComparison.OrdinalIgnoreCase))
.Select(role => role.Key)
.ToList(); // ToList() is required as we are going to modify the same source collection below
IEnumerable <string> keysWithCurrentResourceGroupScopeForRoleDefinitions = this.roleDefinitionsToAssign
.Where(role => role.Value.Item1.Equals(CURRENT_RESOURCE_GROUP_SCOPE, System.StringComparison.OrdinalIgnoreCase))
.Select(role => role.Key)
.ToList();
if (!keysWithCurrentResourceGroupScopeForRoles.Any() &&
!keysWithCurrentResourceGroupScopeForRoleDefinitions.Any())
{
return(false);
}
var resourceGroup = await scaleSet.Manager.ResourceManager
.ResourceGroups
.GetByNameAsync(scaleSet.ResourceGroupName, cancellationToken);
foreach (string key in keysWithCurrentResourceGroupScopeForRoles)
{
rolesToAssign[key] = new System.Tuple <string, BuiltInRole>(resourceGroup.Id, rolesToAssign[key].Item2);
}
foreach (string key in keysWithCurrentResourceGroupScopeForRoleDefinitions)
{
roleDefinitionsToAssign[key] = new System.Tuple <string, string>(resourceGroup.Id, roleDefinitionsToAssign[key].Item2);
}
return(true);
}
/// <summary>
/// Disables the over provisioning for the scale set.
/// </summary>
/// <returns>The over provisioning.</returns>
/// <param name="scaleSet">Scale set.</param>
/// <param name="log">Log.</param>
public static Task DisableOverProvisioning(IVirtualMachineScaleSet scaleSet, ILogger log)
{
if (scaleSet.OverProvisionEnabled)
{
log.LogInformation("Overprovisioning is ON, turning it off");
scaleSet.Inner.Overprovision = false;
return(scaleSet.Update().ApplyAsync());
}
return(Task.CompletedTask);
}
private static bool IsIdentityAssigned(IVirtualMachineScaleSet scaleSet, IIdentity identity)
{
bool isAssigned = scaleSet.ManagedServiceIdentityType == ResourceIdentityType.SystemAssignedUserAssigned ||
scaleSet.ManagedServiceIdentityType == ResourceIdentityType.UserAssigned;
if (!isAssigned)
{
return(false);
}
return(scaleSet.UserAssignedManagedServiceIdentityIds != null &&
scaleSet.UserAssignedManagedServiceIdentityIds.Contains(identity.Id));
}
private async Task Prepare(IAzure azure, OperationPhase phase, IResourceGroup resourceGroup,
IVirtualMachineScaleSet scaleSet, string identityName)
{
var identity = await FindIdentity(azure, resourceGroup, identityName);
var isAssigned = identity != null && IsIdentityAssigned(scaleSet, identity);
switch (phase)
{
case OperationPhase.IdentityNotCreated:
if (isAssigned)
{
await scaleSet.Update()
.WithoutUserAssignedManagedServiceIdentity(identity.Id)
.ApplyAsync();
}
if (identity != null)
{
await azure.Identities.DeleteByIdAsync(identity.Id);
}
break;
case OperationPhase.IdentityCreatedAndNotAssigned:
if (identity == null)
{
identity = await CreateIdentity(azure, resourceGroup, identityName);
}
if (isAssigned)
{
await scaleSet.Update()
.WithoutUserAssignedManagedServiceIdentity(identity.Id)
.ApplyAsync();
}
break;
case OperationPhase.IdentityAssigned:
if (identity == null)
{
identity = await CreateIdentity(azure, resourceGroup, identityName);
}
if (!isAssigned)
{
await scaleSet.Update()
.WithExistingUserAssignedManagedServiceIdentity(identity)
.ApplyAsync();
}
break;
}
}
/// <summary>
/// If there is a failed VM in VMSS - we can reimage them
/// </summary>
/// <param name="scaleSet"></param>
private static async void HouseKeeping(IVirtualMachineScaleSet scaleSet)
{
var failedVms = scaleSet.VirtualMachines.List().Where(vm =>
vm.Inner.ProvisioningState.Equals("Failed", StringComparison.OrdinalIgnoreCase)).ToArray();
if (!failedVms.Any())
{
return;
}
Console.WriteLine("We have some failed VMs and will try to reimage them async");
foreach (var virtualMachineScaleSetVm in failedVms)
{
await virtualMachineScaleSetVm.ReimageAsync();
}
}
/// <summary>
/// This method will perform all changes to scale set
/// </summary>
private static void DeallocationWorkWithScaleSet(
ScaleSetVirtualMachineStripped[] virtualMachinesStripped,
JobRequest[] executingJobs,
IVirtualMachineScaleSet scaleSet, int agentsLimit)
{
Console.WriteLine("Deallocating VMs");
//we need to downscale, only running VMs shall be selected here
var vmInstancesCollection = Decisions.CollectInstanceIdsToDeallocate(virtualMachinesStripped.Where(vm => vm.VmInstanceState.Equals(PowerState.Running)), executingJobs);
DeallocateVms(vmInstancesCollection, scaleSet, agentsLimit);
//if we are deprovisioning - it is some time to do some housekeeping as well
if (Properties.IsDryRun)
{
return;
}
HouseKeeping(scaleSet);
}
private static void DeallocateVms(IEnumerable <ScaleSetVirtualMachineStripped> vmInstances, IVirtualMachineScaleSet scaleSet, int agentsCountToDeallocate)
{
var virtualMachinesCounter = 0;
foreach (var vmInstance in vmInstances)
{
if (virtualMachinesCounter >= agentsCountToDeallocate)
{
break;
}
Console.WriteLine($"Deallocating VM with instance ID {vmInstance}");
virtualMachinesCounter++;
if (Properties.IsDryRun)
{
continue;
}
if (Decisions.IsVmExecutingJob(vmInstance.VmName))
{
//this VM just got job assigned, so we should not deallocate it
continue;
}
scaleSet.VirtualMachines.Inner.BeginDeallocateWithHttpMessagesAsync(Properties.VmScaleSetResourceGroupName, Properties.VmScaleSetName,
vmInstance.VmInstanceId);
}
}
/// <summary>
/// Removes VMSS instances based on the ip addresses reported by backend
/// IP Address.
/// </summary>
/// <returns>completion task.</returns>
/// <param name="scaleSet">Scale set.</param>
/// <param name="serverIPs">Server IPs</param>
/// <param name="log">Log.</param>
public static bool RemoveVMSSInstancesByIP(IVirtualMachineScaleSet scaleSet, List <string> serverIPs, ILogger log)
{
try
{
// first a bit of cleanup, remove all super old pending delete info to prevent leakage
foreach (var k in RecentPendingVMDeleteOperations.Keys.ToList())
{
if (RecentPendingVMDeleteOperations[k] < DateTime.UtcNow - TimeSpan.FromMinutes(20))
{
log.LogInformation($"Cleaning up old pending delete info for vm {k}");
}
}
log.LogInformation("Enumerating VM Instances in ScaleSet");
var vms = scaleSet.VirtualMachines.List().ToList();
// only consider nodes which have been prtovisioned completely for removal
var virtualmachines = vms.Where(x => x.Inner.ProvisioningState == "Succeeded").ToList();
log.LogInformation($"{virtualmachines.Count} machines of {vms.Count} are completely provisioned, checking those for App Payload Failed nodes");
List <string> badInstances = new List <string>();
foreach (var vm in virtualmachines)
{
try
{
if (serverIPs.Contains(vm.ListNetworkInterfaces().First().Inner.IpConfigurations.First().PrivateIPAddress))
{
log.LogInformation("App Payload Failed detected: {0}", vm.InstanceId);
badInstances.Add(vm.InstanceId);
}
}
catch (Exception)
{
log.LogError($"Error reading ip config by VM ID {vm.Id}");
}
}
if (badInstances.Count() != 0)
{
var instancesToDelete = new List <string>();
log.LogInformation("Removing App Payload Failed");
foreach (var badVm in badInstances)
{
if (RecentPendingVMDeleteOperations.ContainsKey(badVm))
{
// we have asked for this vm to be deleted before, if it's more than 10 mins ago
// then try again otherwise skip it
if (DateTime.UtcNow - RecentPendingVMDeleteOperations[badVm] < TimeSpan.FromMinutes(10))
{
log.LogInformation($"*** Instance {badVm} has recent delete request ({RecentPendingVMDeleteOperations[badVm]}), skipping deletion");
}
else
{
// we should delete it and update the timestamp
instancesToDelete.Add(badVm);
}
}
else
{
instancesToDelete.Add(badVm);
}
}
foreach (var v in instancesToDelete)
{
RecentPendingVMDeleteOperations[v] = DateTime.UtcNow;
}
if (instancesToDelete.Any())
{
scaleSet.VirtualMachines.DeleteInstancesAsync(instancesToDelete.ToArray());
return(true);
}
else
{
return(false);
}
}
else
{
log.LogInformation("No Running nodes detected to remove, likely because they are already deleting");
return(false);
}
}
catch (Exception e)
{
log.LogError(e, "Error Removing VMs " + e);
throw;
}
}
/// <summary>
/// Scales the scaleset to the target size (taking into account scaling
/// limits)
/// </summary>
/// <returns>The to target size.</returns>
/// <param name="scaleSet">Scale set.</param>
/// <param name="scaleNodeCount">Scale node count.</param>
/// <param name="maxScaleUpCount">Max scale up count.</param>
/// <param name="maxNodes">Max nodes.</param>
/// <param name="scaleUpQuickly">If set to <c>true</c> scale up quickly.</param>
/// <param name="deletedNodes">If set to <c>true</c>, we deleted nodes in this pass.</param>
/// <param name="log">Log.</param>
public static Task ScaleToTargetSize(IVirtualMachineScaleSet scaleSet, int scaleNodeCount, int maxScaleUpCount, int maxNodes, bool scaleUpQuickly, bool deletedNodes, ILogger log)
{
log.LogInformation($"ScaleToTargetSize scaleNodeCount {scaleNodeCount}, max {maxScaleUpCount}, maxNodes {maxNodes}, scaleUpQuickly {scaleUpQuickly}, deletedNodes {deletedNodes}");
List <Task> pendingTasks = new List <Task>();
try
{
if (scaleNodeCount > maxNodes)
{
log.LogInformation($"Scale requested to {scaleNodeCount} which is larger than max ({maxNodes})");
scaleNodeCount = maxNodes;
}
if (scaleNodeCount > scaleSet.Capacity && scaleUpQuickly)
{
// we are scaling up and want to go a fast as possible so scale by chunks
// of maxScaleUpCount at a time until we reach the target
var currentTarget = Math.Min(scaleSet.Capacity + maxScaleUpCount, scaleNodeCount);
do
{
log.LogInformation($"Scale quickly mode: Scaling to {currentTarget}");
scaleSet.Inner.Sku.Capacity = currentTarget;
pendingTasks.Add(scaleSet.Update().ApplyAsync());
if (currentTarget == scaleNodeCount)
{
break;
}
// sleep for a little bit rather than requesting again immediately
log.LogInformation($"Sleeping 5 seconds...");
Thread.Sleep(TimeSpan.FromSeconds(5));
currentTarget = Math.Min(currentTarget + maxScaleUpCount, scaleNodeCount);
} while (currentTarget <= scaleNodeCount);
return(Task.WhenAll(pendingTasks));
}
else
{
// If we are scaling up in slow mode and asking to scale up by more than max
// only scale by max nodes.
if (scaleNodeCount - scaleSet.Capacity > maxScaleUpCount)
{
log.LogInformation($"Scale up request too large, capacity={scaleSet.Capacity}, request = {scaleNodeCount}, scaling by {maxScaleUpCount} only");
scaleNodeCount = scaleSet.Capacity + maxScaleUpCount;
}
if (!deletedNodes && scaleSet.Capacity == scaleNodeCount)
{
log.LogInformation("**** Not setting scaleset size as we didn't delete any nodes this time and capacity matches");
return(Task.CompletedTask);
}
else
{
log.LogInformation($"Changing Capacity from {scaleSet.Capacity} to {scaleNodeCount}");
scaleSet.Inner.Sku.Capacity = scaleNodeCount;
return(scaleSet.Update().ApplyAsync());
}
}
}
catch (Exception e)
{
log.LogInformation("Error Message: " + e.Message);
throw;
}
}
/**
* Azure Compute sample for managing virtual machine scale sets with un-managed disks -
* - Create a virtual machine scale set behind an Internet facing load balancer
* - Install Apache Web servers in virtual machines in the virtual machine scale set
* - List scale set virtual machine instances and SSH collection string
* - Stop a virtual machine scale set
* - Start a virtual machine scale set
* - Update a virtual machine scale set
* - Double the no. of virtual machines
* - Restart a virtual machine scale set
*/
public async static Task RunSampleAsync(IAzure azure)
{
var region = Region.USWestCentral;
var rgName = SdkContext.RandomResourceName("rgCOVS", 15);
var vnetName = SdkContext.RandomResourceName("vnet", 24);
var loadBalancerName1 = SdkContext.RandomResourceName("intlb" + "-", 18);
var publicIpName = "pip-" + loadBalancerName1;
var frontendName = loadBalancerName1 + "-FE1";
var backendPoolName1 = loadBalancerName1 + "-BAP1";
var backendPoolName2 = loadBalancerName1 + "-BAP2";
var httpProbe = "httpProbe";
var httpsProbe = "httpsProbe";
var httpLoadBalancingRule = "httpRule";
var httpsLoadBalancingRule = "httpsRule";
var natPool50XXto22 = "natPool50XXto22";
var natPool60XXto23 = "natPool60XXto23";
var vmssName = SdkContext.RandomResourceName("vmss", 24);
var userName = "******";
var sshKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCfSPC2K7LZcFKEO+/t3dzmQYtrJFZNxOsbVgOVKietqHyvmYGHEC0J2wPdAqQ/63g/hhAEFRoyehM+rbeDri4txB3YFfnOK58jqdkyXzupWqXzOrlKY4Wz9SKjjN765+dqUITjKRIaAip1Ri137szRg71WnrmdP3SphTRlCx1Bk2nXqWPsclbRDCiZeF8QOTi4JqbmJyK5+0UqhqYRduun8ylAwKKQJ1NJt85sYIHn9f1Rfr6Tq2zS0wZ7DHbZL+zB5rSlAr8QyUdg/GQD+cmSs6LvPJKL78d6hMGk84ARtFo4A79ovwX/Fj01znDQkU6nJildfkaolH2rWFG/qttD [email protected]";
var apacheInstallScript = "https://raw.githubusercontent.com/Azure/azure-sdk-for-java/master/azure-samples/src/main/resources/install_apache.sh";
var installCommand = "bash install_apache.sh";
var fileUris = new List <string>();
fileUris.Add(apacheInstallScript);
try
{
//=============================================================
// Create a virtual network with a frontend subnet
Utilities.Log("Creating virtual network with a frontend subnet ...");
Utilities.Log("Creating a public IP address...");
INetwork network = null;
IPublicIPAddress publicIPAddress = null;
await Task.WhenAll(azure.Networks.Define(vnetName)
.WithRegion(region)
.WithNewResourceGroup(rgName)
.WithAddressSpace("172.16.0.0/16")
.DefineSubnet("Front-end")
.WithAddressPrefix("172.16.1.0/24")
.Attach()
.CreateAsync()
.ContinueWith(n =>
{
network = n.Result;
Utilities.Log("Created a virtual network");
// Print the virtual network details
Utilities.PrintVirtualNetwork(network);
return(network);
}),
azure.PublicIPAddresses.Define(publicIpName)
.WithRegion(region)
.WithNewResourceGroup(rgName)
.WithLeafDomainLabel(publicIpName)
.CreateAsync()
.ContinueWith(pip =>
{
publicIPAddress = pip.Result;
Utilities.Log("Created a public IP address");
// Print the virtual network details
Utilities.PrintIPAddress(publicIPAddress);
return(pip);
}));
//=============================================================
// Create an Internet facing load balancer with
// One frontend IP address
// Two backend address pools which contain network interfaces for the virtual
// machines to receive HTTP and HTTPS network traffic from the load balancer
// Two load balancing rules for HTTP and HTTPS to map public ports on the load
// balancer to ports in the backend address pool
// Two probes which contain HTTP and HTTPS health probes used to check availability
// of virtual machines in the backend address pool
// Three inbound NAT rules which contain rules that map a public port on the load
// balancer to a port for a specific virtual machine in the backend address pool
// - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23
Utilities.Log("Creating a Internet facing load balancer with ...");
Utilities.Log("- A frontend IP address");
Utilities.Log("- Two backend address pools which contain network interfaces for the virtual\n"
+ " machines to receive HTTP and HTTPS network traffic from the load balancer");
Utilities.Log("- Two load balancing rules for HTTP and HTTPS to map public ports on the load\n"
+ " balancer to ports in the backend address pool");
Utilities.Log("- Two probes which contain HTTP and HTTPS health probes used to check availability\n"
+ " of virtual machines in the backend address pool");
Utilities.Log("- Two inbound NAT rules which contain rules that map a public port on the load\n"
+ " balancer to a port for a specific virtual machine in the backend address pool\n"
+ " - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23");
var loadBalancer1 = await azure.LoadBalancers.Define(loadBalancerName1)
.WithRegion(region)
.WithExistingResourceGroup(rgName)
.DefinePublicFrontend(frontendName)
.WithExistingPublicIPAddress(publicIPAddress)
.Attach()
// Add two backend one per rule
.DefineBackend(backendPoolName1)
.Attach()
.DefineBackend(backendPoolName2)
.Attach()
// Add two probes one per rule
.DefineHttpProbe(httpProbe)
.WithRequestPath("/")
.WithPort(80)
.Attach()
.DefineHttpProbe(httpsProbe)
.WithRequestPath("/")
.WithPort(443)
.Attach()
// Add two rules that uses above backend and probe
.DefineLoadBalancingRule(httpLoadBalancingRule)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(80)
.WithProbe(httpProbe)
.WithBackend(backendPoolName1)
.Attach()
.DefineLoadBalancingRule(httpsLoadBalancingRule)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(443)
.WithProbe(httpsProbe)
.WithBackend(backendPoolName2)
.Attach()
// Add nat pools to enable direct VM connectivity for
// SSH to port 22 and TELNET to port 23
.DefineInboundNatPool(natPool50XXto22)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPortRange(5000, 5099)
.WithBackendPort(22)
.Attach()
.DefineInboundNatPool(natPool60XXto23)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPortRange(6000, 6099)
.WithBackendPort(23)
.Attach()
.CreateAsync();
// Print load balancer details
Utilities.Log("Created a load balancer");
Utilities.PrintLoadBalancer(loadBalancer1);
//=============================================================
// Create a virtual machine scale set with three virtual machines
// And, install Apache Web servers on them
Utilities.Log("Creating virtual machine scale set with three virtual machines"
+ "in the frontend subnet ...");
var t1 = new DateTime();
IVirtualMachineScaleSet virtualMachineScaleSet = null;
await azure.VirtualMachineScaleSets.Define(vmssName)
.WithRegion(region)
.WithExistingResourceGroup(rgName)
.WithSku(VirtualMachineScaleSetSkuTypes.StandardD3v2)
.WithExistingPrimaryNetworkSubnet(network, "Front-end")
.WithExistingPrimaryInternetFacingLoadBalancer(loadBalancer1)
.WithPrimaryInternetFacingLoadBalancerBackends(backendPoolName1, backendPoolName2)
.WithPrimaryInternetFacingLoadBalancerInboundNatPools(natPool50XXto22, natPool60XXto23)
.WithoutPrimaryInternalLoadBalancer()
.WithPopularLinuxImage(KnownLinuxVirtualMachineImage.UbuntuServer16_04_Lts)
.WithRootUsername(userName)
.WithSsh(sshKey)
.WithNewDataDisk(100)
.WithNewDataDisk(100, 1, CachingTypes.ReadWrite)
.WithNewDataDisk(100, 2, CachingTypes.ReadWrite, StorageAccountTypes.StandardLRS)
.WithCapacity(3)
// Use a VM extension to install Apache Web servers
.DefineNewExtension("CustomScriptForLinux")
.WithPublisher("Microsoft.OSTCExtensions")
.WithType("CustomScriptForLinux")
.WithVersion("1.4")
.WithMinorVersionAutoUpgrade()
.WithPublicSetting("fileUris", fileUris)
.WithPublicSetting("commandToExecute", installCommand)
.Attach()
.CreateAsync()
.ContinueWith(vmss =>
{
virtualMachineScaleSet = vmss.Result;
var t2 = new DateTime();
Utilities.Log("Created a virtual machine scale set with "
+ "3 Linux VMs & Apache Web servers on them: (took "
+ (t2 - t1).TotalSeconds + " seconds) ");
Utilities.Log();
return(virtualMachineScaleSet);
});
//=============================================================
// List virtual machine scale set instance network interfaces and SSH connection string
Utilities.Log("Listing scale set virtual machine instance network interfaces and SSH connection string...");
foreach (var instance in await virtualMachineScaleSet.VirtualMachines.ListAsync())
{
Utilities.Log("Scale set virtual machine instance #" + instance.InstanceId);
Utilities.Log(instance.Id);
var networkInterfaces = instance.ListNetworkInterfaces();
// Pick the first NIC
var networkInterface = networkInterfaces.ElementAt(0);
foreach (var ipConfig in networkInterface.IPConfigurations.Values)
{
if (ipConfig.IsPrimary)
{
var natRules = ipConfig.ListAssociatedLoadBalancerInboundNatRules();
foreach (var natRule in natRules)
{
if (natRule.BackendPort == 22)
{
Utilities.Log("SSH connection string: " + userName + "@" + publicIPAddress.Fqdn + ":" + natRule.FrontendPort);
break;
}
}
break;
}
}
}
//=============================================================
// Stop the virtual machine scale set
Utilities.Log("Stopping virtual machine scale set ...");
await virtualMachineScaleSet.PowerOffAsync();
Utilities.Log("Stopped virtual machine scale set");
//=============================================================
// Deallocate the virtual machine scale set
Utilities.Log("De-allocating virtual machine scale set ...");
await virtualMachineScaleSet.DeallocateAsync();
Utilities.Log("De-allocated virtual machine scale set");
//=============================================================
// Start the virtual machine scale set
Utilities.Log("Starting virtual machine scale set ...");
await virtualMachineScaleSet.StartAsync();
Utilities.Log("Started virtual machine scale set");
//=============================================================
// Update the virtual machine scale set
// - double the no. of virtual machines
Utilities.Log("Updating virtual machine scale set "
+ "- double the no. of virtual machines ...");
await virtualMachineScaleSet.Update()
.WithCapacity(6)
.WithoutDataDisk(0)
.WithoutDataDisk(200)
.ApplyAsync();
Utilities.Log("Doubled the no. of virtual machines in "
+ "the virtual machine scale set");
//=============================================================
// re-start virtual machine scale set
Utilities.Log("re-starting virtual machine scale set ...");
await virtualMachineScaleSet.RestartAsync();
Utilities.Log("re-started virtual machine scale set");
}
finally
{
try
{
Utilities.Log("Deleting Resource Group: " + rgName);
await azure.ResourceGroups.DeleteByNameAsync(rgName);
Utilities.Log("Deleted Resource Group: " + rgName);
}
catch (NullReferenceException)
{
Utilities.Log("Did not create any resourcesinAzure. No clean up is necessary");
}
catch (Exception g)
{
Utilities.Log(g);
}
}
}
/// <summary>
/// Checks the application gateway back end health metrics and deletes App Payload Failed nodes
/// </summary>
/// <returns><c>true</c>, if we deleted nodes, <c>false</c> otherwise.</returns>
/// <param name="appGw">App gateway</param>
/// <param name="scaleSet">Scale set.</param>
/// <param name="minHealthyServers">Minimum healthy servers.</param>
/// <param name="log">Log.</param>
public static bool CheckApplicationGatewayBEHealthAndDeleteBadNodes(ApplicationGatewayBackendHealthInner appGw, IVirtualMachineScaleSet scaleSet, int minHealthyServers, ILogger log)
{
try
{
log.LogInformation("Enumerating Application Gateway Backend Servers");
var healthy = new List <ApplicationGatewayBackendHealthServer>();
var unhealthy = new List <ApplicationGatewayBackendHealthServer>();
foreach (var server in appGw.BackendAddressPools[0].BackendHttpSettingsCollection[0].Servers)
{
if (server.Health.Value == "Healthy")
{
healthy.Add(server);
}
else
{
unhealthy.Add(server);
}
}
List <string> appGwBadIps = new List <string>();
// If we have unhealthy nodes, then delete them
if (unhealthy.Count > 0)
{
log.LogInformation("App Payload Failed node count = {0}, removing nodes", unhealthy.Count);
if (healthy.Count <= 3)
{
var nodeCount = healthy.Count() + unhealthy.Count() + 3;
log.LogInformation("Healthy Node Count <=3, mandatory scale event to Current Healthly Count + UnHealthy Count + 3 nodes. ScaleTarget Set to {0}", nodeCount);
vmScaleSetOperations.ScaleToTargetSize(scaleSet, nodeCount, 10, 100, false, false, log);
}
return(vmScaleSetOperations.RemoveVMSSInstancesByIP(scaleSet, unhealthy.Select(s => s.Address).ToList(), log));
}
return(false);
}
catch (Exception e)
{
log.LogInformation("Error Message: " + e.Message);
log.LogInformation("HResult: " + e.HResult);
log.LogInformation("InnerException:" + e.InnerException);
return(false);
}
}
/// <summary>Convert the virtual machine scale set instance to scale set entity.</summary>
/// <param name="virtualMachineScaleSet">The scale set instance.</param>
/// <returns></returns>
private static VirtualMachineScaleSetCrawlerResponse ConvertToVirtualMachineScaleSetCrawlerResponse(IVirtualMachineScaleSet virtualMachineScaleSet)
{
var virtualMachineScaleSetEntity = new VirtualMachineScaleSetCrawlerResponse(virtualMachineScaleSet.ResourceGroupName, virtualMachineScaleSet.Id.Replace(Delimeters.ForwardSlash, Delimeters.Exclamatory))
{
ResourceName = virtualMachineScaleSet.Name,
RegionName = virtualMachineScaleSet.RegionName,
Id = virtualMachineScaleSet.Id
};
if (virtualMachineScaleSet.VirtualMachines != null && virtualMachineScaleSet.VirtualMachines.List().Any())
{
virtualMachineScaleSetEntity.HasVirtualMachines = true;
}
return(virtualMachineScaleSetEntity);
}
/// <summary>
/// Checks the application gateway back end health metrics and deletes App Payload Failed nodes
/// </summary>
/// <returns><c>true</c>, if we deleted nodes, <c>false</c> otherwise.</returns>
/// <param name="appGw">App gateway</param>
/// <param name="scaleSet">Scale set.</param>
/// <param name="minHealthyServers">Minimum healthy servers.</param>
/// <param name="log">Log.</param>
public static bool CheckApplicationGatewayBEHealthAndReimageBadNodes(ApplicationGatewayBackendHealthInner appGw, IVirtualMachineScaleSet scaleSet, IVirtualMachineScaleSets allScaleSets, int minHealthyServers, ILogger log)
{
try
{
log.LogInformation("Enumerating Application Gateway Backend Servers");
var healthy = new List <ApplicationGatewayBackendHealthServer>();
var unhealthy = new List <ApplicationGatewayBackendHealthServer>();
foreach (var server in appGw.BackendAddressPools[0].BackendHttpSettingsCollection[0].Servers)
{
if (server.Health.Value == "Healthy")
{
healthy.Add(server);
}
else
{
unhealthy.Add(server);
}
}
List <string> appGwBadIps = new List <string>();
// If we have unhealthy nodes, then delete them
if (unhealthy.Count > 0)
{
log.LogInformation("App Payload Failed node count = {0}, reimaging nodes", unhealthy.Count);
return(VmScaleSetOperations.ReplaceVMSSInstancesByIP(scaleSet, allScaleSets, unhealthy.Select(s => s.Address).ToList(), log));
}
return(false);
}
catch (Exception e)
{
log.LogInformation("Error Message: " + e.Message);
log.LogInformation("HResult: " + e.HResult);
log.LogInformation("InnerException:" + e.InnerException);
return(false);
}
}
/// <summary>
/// Creates RBAC role assignments for the virtual machine scale set MSI service principal.
/// </summary>
/// <param name="scaleSet">The virtual machine scale set.</param>
/// <return>An observable that emits the created role assignments.</return>
///GENMHASH:2E7E577AEA0C43B5B8D7B57BEFEF1E29:DDAE54B3372466B701A29E3B6A6362B2
internal async Task <List <Microsoft.Azure.Management.Graph.RBAC.Fluent.IRoleAssignment> > CreateMSIRbacRoleAssignmentsAsync(IVirtualMachineScaleSet scaleSet, CancellationToken cancellationToken = default(CancellationToken))
{
if (!requireSetup)
{
return(new List <IRoleAssignment>());
}
else if (!scaleSet.IsManagedServiceIdentityEnabled)
{
return(new List <IRoleAssignment>());
}
else if (!this.rolesToAssign.Any() && !this.roleDefinitionsToAssign.Any())
{
return(new List <IRoleAssignment>());
}
try
{
var servicePrincipal = await rbacManager
.ServicePrincipals
.GetByIdAsync(scaleSet.Inner.Identity.PrincipalId, cancellationToken);
await ResolveCurrentResourceGroupScopeAsync(scaleSet);
List <IRoleAssignment> roleAssignments = new List <IRoleAssignment>();
var roleAssignments1 = await Task.WhenAll(rolesToAssign.Values.Select(async(scopeAndRole) =>
{
BuiltInRole role = scopeAndRole.Item2;
string scope = scopeAndRole.Item1;
return(await CreateRbacRoleAssignmentIfNotExistsAsync(servicePrincipal, role.ToString(), scope, true, cancellationToken));
}));
roleAssignments.AddRange(roleAssignments1);
var roleAssignments2 = await Task.WhenAll(roleDefinitionsToAssign.Values.Select(async(scopeAndRole) =>
{
string roleDefinition = scopeAndRole.Item2;
string scope = scopeAndRole.Item1;
return(await CreateRbacRoleAssignmentIfNotExistsAsync(servicePrincipal, roleDefinition, scope, false, cancellationToken));
}));
roleAssignments.AddRange(roleAssignments2);
return(roleAssignments.FindAll(roleAssignment => roleAssignment != null));
}
finally
{
Clear();
}
}
private static void AllocateVms(IEnumerable <ScaleSetVirtualMachineStripped> virtualMachinesStripped, IVirtualMachineScaleSet scaleSet)
{
Console.WriteLine("Starting more VMs");
foreach (var virtualMachineStripped in virtualMachinesStripped)
{
Console.WriteLine($"Starting VM {virtualMachineStripped.VmName} with id {virtualMachineStripped.VmInstanceId}");
if (!Properties.IsDryRun)
{
scaleSet.VirtualMachines.Inner.BeginStartWithHttpMessagesAsync(Properties.VmScaleSetResourceGroupName, Properties.VmScaleSetName,
virtualMachineStripped.VmInstanceId);
}
}
}
public async static Task <List <IVirtualMachineScaleSetVM> > GetVirtualMachineScaleSetVMs(this IVirtualMachineScaleSet vmss)
{
var result = new List <IVirtualMachineScaleSetVM>();
var page = await vmss.VirtualMachines.ListAsync();
if (page != null)
{
do
{
foreach (var item in page)
{
result.Add(item);
}
page = await page.GetNextPageAsync();
} while (page != null);
}
return(result);
}
private static void AllocateVms(ScaleSetVirtualMachineStripped[] virtualMachinesStripped, int agentsLimit, IVirtualMachineScaleSet scaleSet)
{
//this counter is needed to not start extra agents, maybe VM is already starting
var virtualMachinesCounter =
virtualMachinesStripped.Count(vm => vm.VmInstanceState.Equals(PowerState.Starting));
Console.WriteLine("Starting more VMs");
foreach (var virtualMachineStripped in virtualMachinesStripped.Where(vm => vm.VmInstanceState.Equals(PowerState.Deallocated)))
{
if (virtualMachinesCounter >= agentsLimit)
{
break;
}
Console.WriteLine($"Starting VM {virtualMachineStripped.VmName} with id {virtualMachineStripped.VmInstanceId}");
if (!Properties.IsDryRun)
{
scaleSet.VirtualMachines.Inner.BeginStartWithHttpMessagesAsync(Properties.VmScaleSetResourceGroupName, Properties.VmScaleSetName,
virtualMachineStripped.VmInstanceId);
}
virtualMachinesCounter++;
}
}
static async Task Main(string[] args)
{
Console.WriteLine("Hello World!");
var creds = new AzureCredentialsFactory().FromFile("my.azureauth");
var azure = await Azure.Configure()
.WithLogLevel(Microsoft.Azure.Management.ResourceManager.Fluent.Core.HttpLoggingDelegatingHandler.Level.BodyAndHeaders)
.Authenticate(creds)
.WithDefaultSubscriptionAsync();
Console.WriteLine(azure.SubscriptionId);
var allvmss = await azure.VirtualMachineScaleSets.ListAsync();
foreach (var vmss in allvmss)
{
Console.WriteLine(vmss.Name);
}
var vmsss = await azure.VirtualMachineScaleSets.ListByResourceGroupAsync("tihuang");
IVirtualMachineScaleSet tihuangvmss = vmsss.Where(v => v.Name == "tihuang-vmss-win").FirstOrDefault();
await tihuangvmss.Update().WithCapacity(1).ApplyAsync();
var vms = await tihuangvmss.VirtualMachines.ListAsync();
Console.WriteLine(vms.Count());
var vmId = vms.First().InstanceId;
Console.WriteLine(vmId);
// var command = new RunCommandInput("RunShellScript");
// command.Script = new List<string>();
// command.Script.Add("set -e");
// command.Script.Add("echo \"hello world\"");
// // command.Script.Add("cat /var/log/azure/custom-script/handler.log");
// command.Script.Add("ls -l");
// // command.Script.Add("rm -r *");
// command.Script.Add("apt-get -y update && apt-get install -y curl");
// command.Script.Add("mkdir /actions-runner && cd /actions-runner");
// command.Script.Add("curl -O -L https://github.com/actions/runner/releases/download/v2.273.5/actions-runner-linux-x64-2.273.5.tar.gz");
// command.Script.Add("tar xzf ./actions-runner-linux-x64-2.273.5.tar.gz");
// command.Script.Add("ls -l");
// command.Script.Add("echo `echo 77u/ewogICJzY2hlbWUiOiAiT0F1dGgiLAogICJkYXRhIjogewogICAgImNsaWVudElkIjogIjZlZGZmMzYyLTg5YjYtNGEwOC1iYTE0LTE0OGUxOTJjZTIyNCIsCiAgICAiYXV0aG9yaXphdGlvblVybCI6ICJodHRwczovL3ZzdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20vX2FwaXMvb2F1dGgyL3Rva2VuL2JmMDhhODVlLTcyNDEtNDg1OC1hZWI4LWFjNzAwNTZhMTZkNCIKICB9Cn0= | base64 -d` > .credentials");
// command.Script.Add("echo `echo 77u/ewogICJkIjogImFRY3ZCaSt1MFQ4UWpTUHo0YUJwOXN3RHhaclZMTEczK2N1NWRGZWlTaFpFNkI0cWI1ZXFTcjNkR3JKekEvTjJvcEc5MlRsdlRFRkNZRzBJZVFKT3NvRE5sYU5Wc1FOeUVCZ1hob21NQ0k3UnJwWitQcmpBbjlHTE0xQUtsUWxZWCtyNzVIZGZNZnlkSnppeTRsRnlVL3dVY3Btc0wyUCtkRjg1Z3p3SEpqZlhoZENuMXNSa09POVRtUEl5bFA0aUtYcnJ5VnUvR0VZdjU4ZkdUdEp0dElCU082TGNOZjZNcGNLdDdOWkdwd0FCc0RwZGZLZ3RRVGdKT3BveDYrMVd2aUZFaXcxUVA2dGg4b0pWSXFpK0lCZXVNRzBENFhIM21GcTBPUmY3N3h3c0t0WE54ZVdMRlh3Yjd6bk5yNUIxY0VKQmt1Ry9ycHAva2FVR2JXbW4wUT09IiwKICAiZHAiOiAiRnVJWEgwV0ZuMWtmMERFaUE5RXdSaWxKTzZVVk5nRjdqT2xiWjJvVmdqV1BOQzNvbUttY1poQ3FpSDRtbExVVkpZMDF6MVNPUDNpTExoRjZMdGNjSjZ6ZStodUpGWTlZZHpWbitoclR3R21LMjI1WHRKZkNyc3kveVlBWTNGVjBiL2hJS21BTzFPdFJIM1dTdnBQb2dqQVdWU2Z4YWVjNityVXBzLytwWFNVPSIsCiAgImRxIjogImhrRDBSZnJPV2N3eEFtNDhiS011aU9DNzVqdlhmYUh6UXB2QlhkblV3cEg4L0FmdlhMVVhSb1hlbG8yWVU5K2k1YU5RczM5Mi9ubXB2OHQwK1Y2Z3JTTkJ2enk1MXZ1UndzdTNsdTEvMkFkc1E3KzVjdHR3cHU2VWsrN1pNQXl5YlhmUUtCay9aVFMzWHQ1aTVIVVN2RWRMaUxhaTRENVhmaWt5SjJPQnpqRT0iLAogICJleHBvbmVudCI6ICJBUUFCIiwKICAiaW52ZXJzZVEiOiAiQThRVUEvTjFPM0N2UGNHRXREZURUMVJ6WnFienY0RC9oL3o2c2RIZmZXZFNmWDlCS2RQeVM2TmZjYTFoZFcxWGJ2b2c4NkNhVHRkaU9sUDJtbmI3MktKTVU2dmdEekxJeHJrRWtuTXYyRktudzh6ZG5QQVZSTXpDZHEvTnY4bnNrSmovSjZ3c0FEV09CeW1sWGZKNHVsTHZic2h4cXIrSDVpODIzWUtLcXVNPSIsCiAgIm1vZHVsdXMiOiAiczR1UnY0MWtXZEF2MXdDMVBiLzhUd1hZd244WC9WSlJ6REZvemtRQ1ZkWi9vQk9Sc29wT1h3emhUb1RvMHh3TEVsdjYrTFc5M04yMjhkaWZFeGkyUUVhRUtwdi9oWU5uR3lpdktJSEcxeHJ5SUdvU3FsYmw4T2poMThoRWtyR0thbDdINlRZR3dPTjE5WWRMeHh2ZXVuRWltZTd2N3g3RFJQdi8xRmF4QXhkeDBrVGEyOTFoaDVnTVV0TWh3NHJuWVZuYldPMEI3VXpNamdLQ3dSUCs0ek8wUkl5QWdsR0xzTDlENmpEaGVOMGo5ZVUvZHRHckpjd1FVKzB2YnhNWjN1TEJwSjlVUlR3K2ZNd0NuOUcvVjFKODZtYUx0dHlBSW1hZnd4N0taRUJwM25EM3dKL29hZENOTGJoRFVSUmZxTjIxT0FGQzN1ZjU2V01PcFlMNlNRPT0iLAogICJwIjogIjdkdWV4Vys3L0RheXlWaTVySkhaUFJiVnNHNTVYWFI0bXlPZkVuUVVlY1V1OTNwZUR4elVsU0xtNmZzQ1JKcmFUa3ZtcjVTMTVySXc2UllPdGdpbDd6dHRvRnhoQVBPSk9CRXV0cjZxR0N1b3Fac0cwWkhHWGU1N3hHUDZmdHNLMFV1bCtFdG9XUmFlRHFVdUZ4YTlzeDlvTWtiL0pCODMvUlVXNkE4bzFxVT0iLAogICJxIjogIndUMVg4MzJWajB2SWsrTkw5cnRtM2owblg2V3JMVjZxQUt0TndYOG1HWXFPY0VCTGFWZ2RUVWZ0b05qaVdDaktFN0RmT2RKbnFuUDFRaFRhTFZDQWh2WE9UOVJyeXBoTjVTSVYxSEdRNHhpOWhBOHY4S1VCQm43RmhMc1pFZDROOUsyRVdldnZ0WEtGUzBLbGxjSXhucTBVV211elpOUDhlTkdEL3NXOFo5VT0iCn0= | base64 -d` > .credentials_rsaparams");
// command.Script.Add("echo `echo 77u/ewogICJhZ2VudElkIjogNzQsCiAgImFnZW50TmFtZSI6ICJodGwtbWFjIiwKICAicG9vbElkIjogMSwKICAicG9vbE5hbWUiOiAiRGVmYXVsdCIsCiAgInNlcnZlclVybCI6ICJodHRwczovL3BpcGVsaW5lcy5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbS9uWkpYY2drdnVwdDJsMUt1ZGxNczBXdk5OdWM0N3dMSUlqZkpIS3FwZmU1d1FsVVB6biIsCiAgImdpdEh1YlVybCI6ICJodHRwczovL2dpdGh1Yi5jb20vVGluZ2x1b0h1YW5nL21haW4iLAogICJ3b3JrRm9sZGVyIjogIl93b3JrIgp9 | base64 -d` > .runner");
// command.Script.Add("ls -l -a");
// command.Script.Add("cat .runner");
// command.Script.Add("cat .credentials");
// command.Script.Add("cat .credentials_rsaparams");
// command.Script.Add("cp ./bin/runsvc.sh ./runsvc.sh");
// command.Script.Add("chmod u+x ./runsvc.sh");
// command.Script.Add("nohup ./runsvc.sh interactive &");
var command = new RunCommandInput("RunPowerShellScript");
command.Script = new List <string>();
command.Script.Add("if (Test-Path \"\\actions-runner\") { Remove-Item \"\\actions-runner\" -Recurse; }");
command.Script.Add("mkdir \\actions-runner ; cd \\actions-runner");
command.Script.Add($"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; Invoke-WebRequest -Uri https://github.com/actions/runner/releases/download/v2.273.5/actions-runner-win-x64-2.273.5.zip -OutFile actions-runner-win-x64-2.273.5.zip");
command.Script.Add($"Add-Type -AssemblyName System.IO.Compression.FileSystem ; [System.IO.Compression.ZipFile]::ExtractToDirectory(\"$PWD\\actions-runner-win-x64-2.273.5.zip\", \"$PWD\")");
command.Script.Add($"[System.IO.File]::WriteAllText(\"$PWD\\.credentials\", [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String(\"ewogICJzY2hlbWUiOiAiT0F1dGgiLAogICJkYXRhIjogewogICAgImNsaWVudElkIjogIjZlZGZmMzYyLTg5YjYtNGEwOC1iYTE0LTE0OGUxOTJjZTIyNCIsCiAgICAiYXV0aG9yaXphdGlvblVybCI6ICJodHRwczovL3ZzdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20vX2FwaXMvb2F1dGgyL3Rva2VuL2JmMDhhODVlLTcyNDEtNDg1OC1hZWI4LWFjNzAwNTZhMTZkNCIKICB9Cn0=\")))");
command.Script.Add($"$bytes=[System.Convert]::FromBase64String(\"ewogICJkIjogImFRY3ZCaSt1MFQ4UWpTUHo0YUJwOXN3RHhaclZMTEczK2N1NWRGZWlTaFpFNkI0cWI1ZXFTcjNkR3JKekEvTjJvcEc5MlRsdlRFRkNZRzBJZVFKT3NvRE5sYU5Wc1FOeUVCZ1hob21NQ0k3UnJwWitQcmpBbjlHTE0xQUtsUWxZWCtyNzVIZGZNZnlkSnppeTRsRnlVL3dVY3Btc0wyUCtkRjg1Z3p3SEpqZlhoZENuMXNSa09POVRtUEl5bFA0aUtYcnJ5VnUvR0VZdjU4ZkdUdEp0dElCU082TGNOZjZNcGNLdDdOWkdwd0FCc0RwZGZLZ3RRVGdKT3BveDYrMVd2aUZFaXcxUVA2dGg4b0pWSXFpK0lCZXVNRzBENFhIM21GcTBPUmY3N3h3c0t0WE54ZVdMRlh3Yjd6bk5yNUIxY0VKQmt1Ry9ycHAva2FVR2JXbW4wUT09IiwKICAiZHAiOiAiRnVJWEgwV0ZuMWtmMERFaUE5RXdSaWxKTzZVVk5nRjdqT2xiWjJvVmdqV1BOQzNvbUttY1poQ3FpSDRtbExVVkpZMDF6MVNPUDNpTExoRjZMdGNjSjZ6ZStodUpGWTlZZHpWbitoclR3R21LMjI1WHRKZkNyc3kveVlBWTNGVjBiL2hJS21BTzFPdFJIM1dTdnBQb2dqQVdWU2Z4YWVjNityVXBzLytwWFNVPSIsCiAgImRxIjogImhrRDBSZnJPV2N3eEFtNDhiS011aU9DNzVqdlhmYUh6UXB2QlhkblV3cEg4L0FmdlhMVVhSb1hlbG8yWVU5K2k1YU5RczM5Mi9ubXB2OHQwK1Y2Z3JTTkJ2enk1MXZ1UndzdTNsdTEvMkFkc1E3KzVjdHR3cHU2VWsrN1pNQXl5YlhmUUtCay9aVFMzWHQ1aTVIVVN2RWRMaUxhaTRENVhmaWt5SjJPQnpqRT0iLAogICJleHBvbmVudCI6ICJBUUFCIiwKICAiaW52ZXJzZVEiOiAiQThRVUEvTjFPM0N2UGNHRXREZURUMVJ6WnFienY0RC9oL3o2c2RIZmZXZFNmWDlCS2RQeVM2TmZjYTFoZFcxWGJ2b2c4NkNhVHRkaU9sUDJtbmI3MktKTVU2dmdEekxJeHJrRWtuTXYyRktudzh6ZG5QQVZSTXpDZHEvTnY4bnNrSmovSjZ3c0FEV09CeW1sWGZKNHVsTHZic2h4cXIrSDVpODIzWUtLcXVNPSIsCiAgIm1vZHVsdXMiOiAiczR1UnY0MWtXZEF2MXdDMVBiLzhUd1hZd244WC9WSlJ6REZvemtRQ1ZkWi9vQk9Sc29wT1h3emhUb1RvMHh3TEVsdjYrTFc5M04yMjhkaWZFeGkyUUVhRUtwdi9oWU5uR3lpdktJSEcxeHJ5SUdvU3FsYmw4T2poMThoRWtyR0thbDdINlRZR3dPTjE5WWRMeHh2ZXVuRWltZTd2N3g3RFJQdi8xRmF4QXhkeDBrVGEyOTFoaDVnTVV0TWh3NHJuWVZuYldPMEI3VXpNamdLQ3dSUCs0ek8wUkl5QWdsR0xzTDlENmpEaGVOMGo5ZVUvZHRHckpjd1FVKzB2YnhNWjN1TEJwSjlVUlR3K2ZNd0NuOUcvVjFKODZtYUx0dHlBSW1hZnd4N0taRUJwM25EM3dKL29hZENOTGJoRFVSUmZxTjIxT0FGQzN1ZjU2V01PcFlMNlNRPT0iLAogICJwIjogIjdkdWV4Vys3L0RheXlWaTVySkhaUFJiVnNHNTVYWFI0bXlPZkVuUVVlY1V1OTNwZUR4elVsU0xtNmZzQ1JKcmFUa3ZtcjVTMTVySXc2UllPdGdpbDd6dHRvRnhoQVBPSk9CRXV0cjZxR0N1b3Fac0cwWkhHWGU1N3hHUDZmdHNLMFV1bCtFdG9XUmFlRHFVdUZ4YTlzeDlvTWtiL0pCODMvUlVXNkE4bzFxVT0iLAogICJxIjogIndUMVg4MzJWajB2SWsrTkw5cnRtM2owblg2V3JMVjZxQUt0TndYOG1HWXFPY0VCTGFWZ2RUVWZ0b05qaVdDaktFN0RmT2RKbnFuUDFRaFRhTFZDQWh2WE9UOVJyeXBoTjVTSVYxSEdRNHhpOWhBOHY4S1VCQm43RmhMc1pFZDROOUsyRVdldnZ0WEtGUzBLbGxjSXhucTBVV211elpOUDhlTkdEL3NXOFo5VT0iCn0=\")");
command.Script.Add($"Add-Type -AssemblyName System.Security");
command.Script.Add($"$encryptedBytes=[System.Security.Cryptography.ProtectedData]::Protect($bytes, $null, [System.Security.Cryptography.DataProtectionScope]::LocalMachine)");
command.Script.Add($"[System.IO.File]::WriteAllBytes(\"$PWD\\.credentials_rsaparams\", $encryptedBytes)");
command.Script.Add($"[System.IO.File]::WriteAllLines(\"$PWD\\.runner\", [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String(\"ewogICJhZ2VudElkIjogNzQsCiAgImFnZW50TmFtZSI6ICJodGwtbWFjIiwKICAicG9vbElkIjogMSwKICAicG9vbE5hbWUiOiAiRGVmYXVsdCIsCiAgInNlcnZlclVybCI6ICJodHRwczovL3BpcGVsaW5lcy5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbS9uWkpYY2drdnVwdDJsMUt1ZGxNczBXdk5OdWM0N3dMSUlqZkpIS3FwZmU1d1FsVVB6biIsCiAgImdpdEh1YlVybCI6ICJodHRwczovL2dpdGh1Yi5jb20vVGluZ2x1b0h1YW5nL21haW4iLAogICJ3b3JrRm9sZGVyIjogIl93b3JrIgp9\")))");
command.Script.Add("cmd /c start $PWD\\run.cmd");
Console.WriteLine(string.Join(Environment.NewLine, command.Script));
// var runResult = await tihuangvmss.RunCommandVMInstanceAsync(vmId, command);
// foreach (var res in runResult.Value)
// {
// Console.WriteLine($"${res.Code}: {res.Message}");
// }
// Console.WriteLine(runResult.Value.First().Message);
}
