static void CheckInputArgs(string[] args)
{
bool invalidOptions = false;
// parse args
var agentConfig = new ArgsOption();
var result = Parser.Default.ParseArguments <ArgsOption>(args)
.WithParsed(options => agentConfig = options)
.WithNotParsed(error => {
Console.WriteLine($"Fail to parse the options: {error}");
invalidOptions = true;
});
if (invalidOptions)
{
return;
}
if (agentConfig.BenchClientListFile != null)
{
Util.Log($"Bench client output file: {agentConfig.BenchClientListFile}");
}
if (agentConfig.VMHostFile != null)
{
Util.Log($"VM host output file: {agentConfig.VMHostFile}");
}
Util.Log($"auth file: {agentConfig.AuthFile}");
var credentials = SdkContext.AzureCredentialsFactory
.FromFile(agentConfig.AuthFile);
var azure = Azure
.Configure()
.WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic)
.Authenticate(credentials)
.WithDefaultSubscription();
var img = azure.VirtualMachineCustomImages.GetByResourceGroup(agentConfig.ImgResourceGroup, agentConfig.ImgName);
Util.Log($"Customized image id: {img.Id}");
var region = img.Region;
Util.Log($"target region: {region.Name}");
var VmSize = VirtualMachineSizeTypes.Parse(agentConfig.VmSize);
Util.Log($"VM size: {VmSize}");
if (agentConfig.SshPubKeyFile != null)
{
var sshPubKey = System.IO.File.ReadAllText(agentConfig.SshPubKeyFile);
Util.Log($"SSH public key: {sshPubKey}");
}
Util.Log($"Accelerated Network: {agentConfig.AcceleratedNetwork}");
}
static void CheckVMPorts(string[] args)
{
bool invalidOptions = false;
// parse args
var agentConfig = new ArgsOption();
var result = Parser.Default.ParseArguments <ArgsOption>(args)
.WithParsed(options => agentConfig = options)
.WithNotParsed(error => {
Console.WriteLine($"Fail to parse the options: {error}");
invalidOptions = true;
});
if (invalidOptions)
{
return;
}
var sw = new Stopwatch();
sw.Start();
// auth file
Util.Log($"auth file: {agentConfig.AuthFile}");
var credentials = SdkContext.AzureCredentialsFactory
.FromFile(agentConfig.AuthFile);
var azure = Azure
.Configure()
.WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic)
.Authenticate(credentials)
.WithDefaultSubscription();
var pubIP = azure.PublicIPAddresses.GetByResourceGroup("honzhanautovm0705", "hzautovm07050PubIP");
Util.Log($"Public IP: {pubIP.IPAddress}");
CheckAllPorts(pubIP.IPAddress);
}
static async Task CreateVM(string[] args)
{
bool invalidOptions = false;
// parse args
var agentConfig = new ArgsOption();
var result = Parser.Default.ParseArguments <ArgsOption>(args)
.WithParsed(options => agentConfig = options)
.WithNotParsed(error => {
Util.Log($"Fail to parse the options: {error}");
invalidOptions = true;
});
if (invalidOptions)
{
Util.Log("Invalid options");
return;
}
var sw = new Stopwatch();
sw.Start();
// auth file
Util.Log($"auth file: {agentConfig.AuthFile}");
var credentials = SdkContext.AzureCredentialsFactory
.FromFile(agentConfig.AuthFile);
var azure = Azure
.Configure()
.WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic)
.Authenticate(credentials)
.WithDefaultSubscription();
var img = await Util.GetVMImageWithRetry(azure,
agentConfig.ImgResourceGroup,
agentConfig.ImgName,
agentConfig.MaxRetry);
if (img == null)
{
throw new Exception("Fail to get custom image");
}
Util.Log($"Customized image id: {img.Id}");
var region = img.Region;
Util.Log($"target region: {region.Name}");
var VmSize = VirtualMachineSizeTypes.Parse(agentConfig.VmSize);
Util.Log($"VM size: {VmSize}");
string sshPubKey = null;
if (agentConfig.SshPubKeyFile == null && agentConfig.VmType == 1)
{
Util.Log("SSH public key is not set for Linux VM");
throw new Exception("SSH public key is not specified!");
}
else if (agentConfig.SshPubKeyFile != null && agentConfig.VmType == 1)
{
sshPubKey = File.ReadAllText(agentConfig.SshPubKeyFile);
Util.Log($"SSH public key: {sshPubKey}");
Util.Log($"Accelerated Network: {agentConfig.AcceleratedNetwork}");
}
if (agentConfig.VmType == 2 && agentConfig.Password == null)
{
Util.Log($"You must specify password for windows VM by -p XXXX");
return;
}
var resourceGroupName = agentConfig.ResourceGroup;
if (!azure.ResourceGroups.Contain(resourceGroupName))
{
await azure.ResourceGroups.Define(resourceGroupName).WithRegion(region).CreateAsync();
}
INetwork network = null;
string subNetName = null;
if (!string.IsNullOrEmpty(agentConfig.VirtualNetwork))
{
Util.Log("Query virtual network...");
network = await Util.GetVirtualNetworkAsync(azure, resourceGroupName, agentConfig.VirtualNetwork);
if (!string.IsNullOrEmpty(agentConfig.Subnet))
{
var sub = agentConfig.Subnet;
foreach (var k in network.Subnets.Keys)
{
if (k == sub)
{
subNetName = k;
break;
}
}
if (subNetName == null)
{
throw new Exception($"The input '{agentConfig.Subnet}' is invalid");
}
}
else
{
Util.Log("No subnet is specified, the system will select one");
foreach (var k in network.Subnets.Keys)
{
subNetName = k;
break;
}
}
if (subNetName == null)
{
throw new Exception($"No valid subnet is found");
}
}
else
{
// create virtual net
Util.Log("Creating virtual network...");
subNetName = agentConfig.Prefix + "Subnet";
network = await Util.CreateVirtualNetworkWithRetry(azure,
subNetName,
resourceGroupName,
agentConfig.Prefix + "VNet",
region,
agentConfig.MaxRetry);
}
if (network == null)
{
throw new Exception("Fail to create virtual network");
}
Util.Log("Finish creating virtual network");
// Prepare a batch of Creatable Virtual Machines definitions
var creatableVirtualMachines = new List <ICreatable <IVirtualMachine> >();
// create vms
Util.Log("Creating public IP address...");
var publicIpTaskList = await Util.CreatePublicIPAddrListWithRetry(azure,
agentConfig.VmCount,
agentConfig.Prefix,
resourceGroupName,
region,
agentConfig.MaxRetry);
if (publicIpTaskList == null)
{
throw new Exception("Fail to create Public IP Address");
}
Util.Log("Finish creating public IP address...");
Util.Log($"Creating network security group...");
var nsg = await Util.CreateNetworkSecurityGroupWithRetry(azure,
resourceGroupName,
agentConfig.Prefix + "NSG",
agentConfig,
region);
if (nsg == null)
{
throw new Exception("Fail to create network security group");
}
Util.Log($"Finish creating network security group...");
Util.Log("Creating network interface...");
var nicTaskList = await Util.CreateNICWithRetry(azure,
resourceGroupName,
agentConfig,
network,
publicIpTaskList,
subNetName,
nsg,
region);
if (nicTaskList == null)
{
throw new Exception("Fail to create NIC task list");
}
Util.Log("Finish creating network interface...");
if (agentConfig.VmType == 1)
{
for (var i = 0; i < agentConfig.VmCount; i++)
{
var vmName = agentConfig.VmCount == 1 ? agentConfig.Prefix : agentConfig.Prefix + Convert.ToString(i);
var vm = azure.VirtualMachines.Define(vmName)
.WithRegion(region)
.WithExistingResourceGroup(resourceGroupName)
.WithExistingPrimaryNetworkInterface(nicTaskList[i].Result)
.WithLinuxCustomImage(img.Id)
.WithRootUsername(agentConfig.Username)
.WithSsh(sshPubKey)
.WithComputerName(agentConfig.Prefix + Convert.ToString(i))
.WithSize(VmSize);
creatableVirtualMachines.Add(vm);
}
}
else if (agentConfig.VmType == 2)
{
for (var i = 0; i < agentConfig.VmCount; i++)
{
var vmName = agentConfig.VmCount == 1 ? agentConfig.Prefix : agentConfig.Prefix + Convert.ToString(i);
var vm = azure.VirtualMachines.Define(vmName)
.WithRegion(region)
.WithExistingResourceGroup(resourceGroupName)
.WithExistingPrimaryNetworkInterface(nicTaskList[i].Result)
.WithWindowsCustomImage(img.Id)
.WithAdminUsername(agentConfig.Username)
.WithAdminPassword(agentConfig.Password)
.WithComputerName(agentConfig.Prefix + Convert.ToString(i))
.WithSize(VmSize);
creatableVirtualMachines.Add(vm);
}
}
Util.Log("Ready to create virtual machine...");
sw.Stop();
Util.Log($"prepare for creating vms elapsed time: {sw.Elapsed.TotalMinutes} min");
sw.Restart();
Util.Log($"Creating vms");
var virtualMachines = azure.VirtualMachines.Create(creatableVirtualMachines.ToArray());
Util.Log($"Finish creating vms");
Util.Log("Check SSH port");
var portCheckTaskList = new List <Task>();
for (var i = 0; i < agentConfig.VmCount; i++)
{
var publicIPAddress = azure.PublicIPAddresses
.GetByResourceGroup(resourceGroupName,
agentConfig.Prefix + Convert.ToString(i) + "PubIP");
portCheckTaskList.Add(Task.Run(async() => await WaitPortOpen(publicIPAddress.IPAddress, agentConfig.SshPort)));
}
if (Task.WaitAll(portCheckTaskList.ToArray(), TimeSpan.FromSeconds(120)))
{
Util.Log("All ports are ready");
}
else
{
Util.Log("Not all ports are ready");
}
sw.Stop();
Util.Log($"creating vms elapsed time: {sw.Elapsed.TotalMinutes} minutes");
if (agentConfig.VMHostFile != null)
{
var builder = new StringBuilder();
for (var i = 0; i < agentConfig.VmCount; i++)
{
if (i != 0)
{
builder.Append('|');
}
builder.Append(agentConfig.Prefix).Append(i).Append(".")
.Append(region.Name).Append(".cloudapp.azure.com");
}
File.WriteAllText(agentConfig.VMHostFile, builder.ToString());
}
if (agentConfig.BenchClientListFile != null)
{
var builder = new StringBuilder();
for (var i = 0; i < agentConfig.VmCount; i++)
{
if (i != 0)
{
builder.Append('|');
}
builder.Append(agentConfig.Prefix).Append(i).Append(".")
.Append(region.Name).Append(".cloudapp.azure.com")
.Append(':').Append(agentConfig.SshPort).Append(':').Append(agentConfig.Username);
}
File.WriteAllText(agentConfig.BenchClientListFile, builder.ToString());
}
}
static async Task <List <Task <INetworkInterface> > > CreateNICWithRetry(IAzure azure, string resourceGroupName,
ArgsOption agentConfig, INetwork network, List <Task <IPublicIPAddress> > publicIpTaskList,
string subNetName, INetworkSecurityGroup nsg, Region region)
{
var j = 0;
var i = 0;
var maxTry = agentConfig.MaxRetry;
var nicTaskList = new List <Task <INetworkInterface> >();
while (j < maxTry)
{
try
{
var allowAcceleratedNet = false;
if (agentConfig.CandidateOfAcceleratedNetVM != null)
{
allowAcceleratedNet = CheckValidVMForAcceleratedNet(agentConfig.CandidateOfAcceleratedNetVM, agentConfig.VmSize);
}
for (i = 0; i < agentConfig.VmCount; i++)
{
var nicName = agentConfig.Prefix + Convert.ToString(i) + "NIC";
Task <INetworkInterface> networkInterface = null;
if (allowAcceleratedNet && agentConfig.AcceleratedNetwork)
{
networkInterface = azure.NetworkInterfaces.Define(nicName)
.WithRegion(region)
.WithExistingResourceGroup(resourceGroupName)
.WithExistingPrimaryNetwork(network)
.WithSubnet(subNetName)
.WithPrimaryPrivateIPAddressDynamic()
.WithExistingPrimaryPublicIPAddress(publicIpTaskList[i].Result)
.WithExistingNetworkSecurityGroup(nsg)
.WithAcceleratedNetworking()
.CreateAsync();
Util.Log("Accelerated Network is enabled!");
}
else
{
Util.Log("Accelerated Network is disabled!");
networkInterface = azure.NetworkInterfaces.Define(nicName)
.WithRegion(region)
.WithExistingResourceGroup(resourceGroupName)
.WithExistingPrimaryNetwork(network)
.WithSubnet(subNetName)
.WithPrimaryPrivateIPAddressDynamic()
.WithExistingPrimaryPublicIPAddress(publicIpTaskList[i].Result)
.WithExistingNetworkSecurityGroup(nsg)
.CreateAsync();
}
nicTaskList.Add(networkInterface);
}
await Task.WhenAll(nicTaskList.ToArray());
return(nicTaskList);
}
catch (Exception e)
{
Util.Log(e.ToString());
nicTaskList.Clear();
var allNICs = azure.NetworkInterfaces.ListByResourceGroupAsync(resourceGroupName);
await allNICs;
var ids = new List <string>();
var enumerator = allNICs.Result.GetEnumerator();
while (enumerator.MoveNext())
{
ids.Add(enumerator.Current.Id);
}
await azure.NetworkInterfaces.DeleteByIdsAsync(ids);
if (j + 1 < maxTry)
{
Util.Log($"Fail to create NIC for {e.Message} and will retry");
}
else
{
Util.Log($"Fail to create NIC for {e.Message} and retry has reached max limit, will return with failure");
}
}
j++;
}
return(null);
}
static INetworkSecurityGroup CreateNetworkSecurityGroupWithRetry(IAzure azure,
string resourceGroupName, string name, ArgsOption agentConfig, Region region)
{
INetworkSecurityGroup rtn = null;
var i = 0;
var maxRetry = agentConfig.MaxRetry;
var azureRegionIP =
"167.220.148.0/23,131.107.147.0/24,131.107.159.0/24,131.107.160.0/24,131.107.174.0/24,167.220.24.0/24,167.220.26.0/24,167.220.238.0/27,167.220.238.128/27,167.220.238.192/27,167.220.238.64/27,167.220.232.0/23,167.220.255.0/25,167.220.242.0/27,167.220.242.128/27,167.220.242.192/27,167.220.242.64/27,94.245.87.0/24,167.220.196.0/23,194.69.104.0/25,191.234.97.0/26,167.220.0.0/23,167.220.2.0/24,207.68.190.32/27,13.106.78.32/27,10.254.32.0/20,10.97.136.0/22,13.106.174.32/27,13.106.4.96/27";
string allowedIpRange = azureRegionIP;
if (agentConfig.BenchServerIP != null)
{
allowedIpRange += "," + agentConfig.BenchServerIP;
}
while (i < maxRetry)
{
try
{
rtn = azure.NetworkSecurityGroups.Define(agentConfig.Prefix + "NSG")
.WithRegion(region)
.WithExistingResourceGroup(resourceGroupName)
.DefineRule("Limit-Benchserver")
.AllowInbound()
.FromAddress(allowedIpRange)
.FromAnyPort()
.ToAnyAddress()
.ToPort(22)
.WithProtocol(SecurityRuleProtocol.Tcp)
.WithPriority(800)
.WithDescription("Limit SSH")
.Attach()
.DefineRule("New-SSH-Port")
.AllowInbound()
.FromAnyAddress()
.FromAnyPort()
.ToAnyAddress()
.ToPort(agentConfig.SshPort)
.WithProtocol(SecurityRuleProtocol.Tcp)
.WithPriority(900)
.WithDescription("New SSH Port")
.Attach()
.DefineRule("Benchmark-Port")
.AllowInbound()
.FromAnyAddress()
.FromAnyPort()
.ToAnyAddress()
.ToPort(agentConfig.OtherPort)
.WithProtocol(SecurityRuleProtocol.Tcp)
.WithPriority(901)
.WithDescription("Benchmark Port")
.Attach()
.DefineRule("Service-Ports")
.AllowInbound()
.FromAnyAddress()
.FromAnyPort()
.ToAnyAddress()
.ToPortRange(5001, 5003)
.WithProtocol(SecurityRuleProtocol.Tcp)
.WithPriority(903)
.WithDescription("Service Port")
.Attach()
.DefineRule("Chat-Sample-Ports")
.AllowInbound()
.FromAnyAddress()
.FromAnyPort()
.ToAnyAddress()
.ToPort(agentConfig.ChatSamplePort)
.WithProtocol(SecurityRuleProtocol.Tcp)
.WithPriority(904)
.WithDescription("Chat Sample Port")
.Attach()
.DefineRule("RDP-Port")
.AllowInbound()
.FromAnyAddress()
.FromAnyPort()
.ToAnyAddress()
.ToPort(agentConfig.RDPPort)
.WithProtocol(SecurityRuleProtocol.Tcp)
.WithPriority(905)
.WithDescription("Windows RDP Port")
.Attach()
.Create();
}
catch (Exception e)
{
Util.Log(e.ToString());
if (i + 1 < maxRetry)
{
Util.Log($"Fail to create security network group for {e.Message} and will retry");
}
else
{
Util.Log($"Fail to create security network group for {e.Message} and retry has reached max limit, will return with failure");
}
}
i++;
}
return(rtn);
}
public static async Task <INetworkSecurityGroup> CreateNetworkSecurityGroupWithRetry(IAzure azure,
string resourceGroupName, string name, ArgsOption agentConfig, Region region)
{
var azureRegionIP = IPRange;
var allowedIpRange = azureRegionIP.Split(',');
INetworkSecurityGroup rtn = null;
var i = 0;
var maxRetry = agentConfig.MaxRetry;
while (i < maxRetry)
{
try
{
rtn = await azure.NetworkSecurityGroups.Define(agentConfig.Prefix + "NSG")
.WithRegion(region)
.WithExistingResourceGroup(resourceGroupName)
.DefineRule("New-SSH-Port")
.AllowInbound()
.FromAddresses(allowedIpRange)
.FromAnyPort()
.ToAnyAddress()
.ToPort(agentConfig.SshPort)
.WithProtocol(SecurityRuleProtocol.Tcp)
.WithPriority(900)
.WithDescription("New SSH Port")
.Attach()
.DefineRule("Benchmark-Port")
.AllowInbound()
.FromAnyAddress()
.FromAnyPort()
.ToAnyAddress()
.ToPort(agentConfig.OtherPort)
.WithProtocol(SecurityRuleProtocol.Tcp)
.WithPriority(901)
.WithDescription("Benchmark Port")
.Attach()
.DefineRule("Service-Ports")
.AllowInbound()
.FromAnyAddress()
.FromAnyPort()
.ToAnyAddress()
.ToPortRange(5001, 5003)
.WithProtocol(SecurityRuleProtocol.Tcp)
.WithPriority(903)
.WithDescription("Service Port")
.Attach()
.DefineRule("Chat-Sample-Ports")
.AllowInbound()
.FromAnyAddress()
.FromAnyPort()
.ToAnyAddress()
.ToPort(agentConfig.ChatSamplePort)
.WithProtocol(SecurityRuleProtocol.Tcp)
.WithPriority(904)
.WithDescription("Chat Sample Port")
.Attach()
.DefineRule("RDP-Port")
.AllowInbound()
.FromAddresses(allowedIpRange)
.FromAnyPort()
.ToAnyAddress()
.ToPort(agentConfig.RDPPort)
.WithProtocol(SecurityRuleProtocol.Tcp)
.WithPriority(905)
.WithDescription("Windows RDP Port")
.Attach()
.DefineRule("Jenkins-Nginx")
.AllowInbound()
.FromAddresses(allowedIpRange)
.FromAnyPort()
.ToAnyAddress()
.ToPortRanges(new string[] { "8080", "8181", "8000" })
.WithProtocol(SecurityRuleProtocol.Tcp)
.WithPriority(906)
.WithDescription("Jenkins and Nginx ports")
.Attach()
.CreateAsync();
}
catch (Exception e)
{
Util.Log(e.ToString());
if (i + 1 < maxRetry)
{
Util.Log($"Fail to create security network group for {e.Message} and will retry");
}
else
{
Util.Log($"Fail to create security network group for {e.Message} and retry has reached max limit, will return with failure");
}
}
i++;
}
return(rtn);
}