public async Task <IActionResult> TestDhmConcurrentVmCreation(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
var parameters = req.GetQueryParameterDictionary();
if (!parameters.ContainsKey(ResourceGroupName) || string.IsNullOrEmpty(parameters[ResourceGroupName]))
{
return(new BadRequestObjectResult("Resource group name was missing in the query parameters."));
}
if (!parameters.ContainsKey(HostGroupName) || string.IsNullOrEmpty(parameters[HostGroupName]))
{
return(new BadRequestObjectResult("Host group name was missing in the query parameters."));
}
if (!parameters.ContainsKey(VmCount) || string.IsNullOrEmpty(parameters[VmCount]))
{
return(new BadRequestObjectResult("VmCount was missing in the query parameters."));
}
if (!parameters.ContainsKey(VmSku) || string.IsNullOrEmpty(parameters[VmSku]))
{
return(new BadRequestObjectResult("VM SKU was missing in the query parameters."));
}
var authEndpoint = _configuration["AuthEndpoint"];
var azureRmEndpoint = _configuration["AzureRmEndpoint"];
var location = _configuration["Location"];
var virtualMachineSize = parameters[VmSku];
var numVirtualMachines = int.Parse(parameters[VmCount]);
var tenantId = _configuration["TenantId"];
var clientId = _configuration["ClientId"];
var clientSecret = _configuration["FairfaxClientSecret"];
var subscriptionId = _configuration["SubscriptionId"];
var resourceGroupName = parameters[ResourceGroupName];
var hostGroupName = parameters[HostGroupName];
log.LogInformation($"Generating auth token...");
var token = await TokenHelper.GetToken(
authEndpoint,
azureRmEndpoint,
tenantId,
clientId,
clientSecret);
var customTokenProvider = new AzureCredentials(
new TokenCredentials(token),
new TokenCredentials(token),
tenantId,
AzureEnvironment.FromName(_configuration["CloudName"]));
var client = RestClient
.Configure()
.WithEnvironment(AzureEnvironment.FromName(_configuration["CloudName"]))
.WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic)
.WithCredentials(customTokenProvider)
.Build();
var azure = Azure.Authenticate(client, tenantId).WithSubscription(subscriptionId);
var computeManagementClient = new ComputeManagementClient(customTokenProvider)
{
SubscriptionId = subscriptionId,
BaseUri = new Uri(_configuration["ResourceManagerUri"]),
LongRunningOperationRetryTimeout = 5
};
log.LogInformation($"Creating resource group ({resourceGroupName}), if needed");
var resourceGroup = azure.ResourceGroups.Define(resourceGroupName)
.WithRegion(location)
.Create();
log.LogInformation($"Creating host group ({hostGroupName}), if needed");
var newDedicatedHostGroup = new DedicatedHostGroup()
{
Location = location,
PlatformFaultDomainCount = 1
};
await computeManagementClient.DedicatedHostGroups.CreateOrUpdateAsync(
resourceGroupName,
hostGroupName,
newDedicatedHostGroup);
var taskList = new List <Task <HttpResponseMessage> >();
var inputDictionary = new Dictionary <string, StringContent>();
for (var i = 0; i < numVirtualMachines; i++)
{
var vmName = $"vm{i}-{new Random().Next(1, 10000)}";
log.LogInformation($"Configuring (not provisioning) VM: {vmName}");
var virtualMachine = CreateVmHelper.CreateVirtualMachine(
computeManagementClient,
azure,
Region.Create(location),
resourceGroupName,
vmName,
virtualMachineSize,
null,
"pip-" + Guid.NewGuid(),
"adh-poc-vnet",
"nic-" + Guid.NewGuid());
#if DEBUG
var createVmUri =
$"http://localhost:7071/api/CreateVm" +
$"?token={token}" +
$"&cloudName={_configuration["CloudName"]}" +
$"&tenantId={tenantId}" +
$"&subscriptionId={subscriptionId}" +
$"&resourceGroup={resourceGroupName}" +
$"&location={location}" +
$"&vmSku={virtualMachineSize}" +
$"&vmName={vmName}" +
$"&dedicatedHostGroupName={hostGroupName}" +
$"&platformFaultDomainCount=1";
#else
var createVmUri =
_configuration["DhmCreateVmnUri"] +
$"&token={token}" +
$"&cloudName={_configuration["CloudName"]}" +
$"&tenantId={tenantId}" +
$"&subscriptionId={subscriptionId}" +
$"&resourceGroup={resourceGroupName}" +
$"&location={location}" +
$"&vmSku={virtualMachineSize}" +
$"&vmName={vmName}" +
$"&dedicatedHostGroupName={hostGroupName}" +
$"&platformFaultDomainCount=1";
#endif
var httpContent = new StringContent(JsonConvert.SerializeObject(virtualMachine), Encoding.UTF8, "application/json");
inputDictionary[createVmUri] = httpContent;
}
foreach (var item in inputDictionary)
{
taskList.Add(_httpClient.PostAsync(item.Key, item.Value));
}
await Task.WhenAll(taskList);
return(new OkObjectResult($"VM provisioning kicked off successfully for {numVirtualMachines} VMs - exiting."));
}
public ICdnManager Authenticate(AzureCredentials credentials, string subscriptionId)
{
return(new CdnManager(BuildRestClient(credentials), subscriptionId));
}
public static void Main(string[] args)
{
Logger.Info("Program startup");
Logger.Info($"Program Name: {AppName}");
_clp = new CommandLineProcessor(args, Logger);
var AppId = "60c73d93-d5ab-42e8-b4d5-1b12c6ec4192";
var AuthKey = "afhuxg33dlxpu/V0lHtLMN9lhpurkCU6jENxtQ7b4uc=";
var TenantID = "373afff1-2527-439e-a138-0cd72c9325e4";
var SubscriptionID = "cf8611e4-c7d7-4eac-a7cf-aa5d6c117499";
var SubscriptionName = "Pay-As-You-Go";
var ResGroups = "PROD-SVC, PROD-WEB, PROD-WEBJOB, Default-Web-EastUS";
var credentials = AzureCredentials.FromServicePrincipal(AppId, AuthKey, TenantID, AzureEnvironment.AzureGlobalCloud);
var azure = Azure
.Configure()
.WithLogLevel(HttpLoggingDelegatingHandler.Level.BASIC)
.Authenticate(credentials)
.WithDefaultSubscription();
//GetResourceGroups(azure);
var fullenchalada = new List <WebAppDto>();
foreach (var grp in ResGroups.Split(','))
{
fullenchalada.AddRange(GetWebApps(azure, grp.Trim()));
}
foreach (WebAppDto dto in fullenchalada)
{
Logger.Info($"App {dto.AppName} URL: {dto.Url}");
}
//var resourceGroupName = "PROD-SVC"; //TEST-SVC";
// var apigatewayresgroup = azure.ResourceGroups.GetByName("Api-Default-East-US-2");
//var export= apigatewayresgroup.ExportTemplate(ResourceGroupExportTemplateOptions.INCLUDE_BOTH);
//var e = apigatewayresgroup.Inner.Name;
//foreach (var gateway in azure.ResourceGroups.GetByName(["Api-Default-East-US-2"]))
//{
// Console.WriteLine($"name: {gateway.Name}");
//}
//azure.WebApps.Define()
Logger.Info("Program End");
// Logger.Error("Something screwed up");
// Exception e = new ArgumentNullException();
// Logger.Error("Something screwed up and here is the exception", e);
Console.ReadLine();
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="credentials">Credentials for the subscription</param>
/// <param name="subscriptionId">Subscription Id</param>
/// <param name="resourceGroup">Name of resource group</param>
public StorageBlobs(AzureCredentials credentials, string subscriptionId, string resourceGroup) : base(credentials, subscriptionId)
{
this.resourceGroup = resourceGroup;
}
public static void Main(string[] args)
{
// Subscription authorization file name example:
// Azure CLI Demo-2d.azureauth
// -2d means that resource groups should be deleted after 2 days.
string[] files = Directory.GetFiles(Environment.GetEnvironmentVariable("AZURE_CLEANUP_AUTH_LOCATION"), "*.azureauth");
Console.WriteLine($"Found {files.Length} subscriptions to clean-up...");
foreach (var file in files)
{
Console.WriteLine($"Cleaning subscription from auth '{file}'");
try
{
//=================================================================
// Authenticate
AzureCredentials credentials = SdkContext.AzureCredentialsFactory.FromFile(file);
var azure = Azure
.Configure()
.WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic)
.Authenticate(credentials)
.WithDefaultSubscription();
var client = new Microsoft.Azure.Management.ResourceManager.ResourceManagementClient(credentials);
client.SubscriptionId = credentials.DefaultSubscriptionId;
Regex r = new Regex(@"-(\d)d", RegexOptions.IgnoreCase);
var matches = r.Matches(Path.GetFileName(file));
int dayTTL = -1;
if (matches.Count > 0)
{
dayTTL = Convert.ToInt32(matches[0].Value[1]) - '0';
Console.WriteLine($" - Resource group TTL for this subscription is '{dayTTL}' day(s).");
}
try
{
foreach (var rg in azure.ResourceGroups.List())
{
try
{
if (rg.Name.EndsWith("-permanent", StringComparison.OrdinalIgnoreCase))
{
Console.WriteLine($" - Resource Group '{rg.Name}' is marked as 'DO NOT DELETE'. Skipping.");
continue;
}
if ("Deleting".Equals(rg.ProvisioningState, StringComparison.OrdinalIgnoreCase))
{
Console.WriteLine($" - Resource Group '{rg.Name}' is already in 'Deleting' state.");
continue;
}
var rgCreationTime = GetRGCreationDateTime(credentials, client, rg);
if (dayTTL != -1)
{
var duration = (DateTime.Now - rgCreationTime).TotalDays;
if (duration < dayTTL)
{
Console.WriteLine($" - Resource Group '{rg.Name}' was created less than {dayTTL} day(s) ago. Skipping.");
continue;
}
}
client.ResourceGroups.BeginDeleteWithHttpMessagesAsync(rg.Name)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
Console.WriteLine($" - Deleted Resource Group '{rg.Name}'.");
}
catch (Exception ex)
{
Console.WriteLine($" [ERROR]: Exception while deleting Resource Group '{rg.Name}'.");
Console.WriteLine(ex);
}
}
}
catch (Exception ex)
{
Console.WriteLine($" [ERROR]: Exception while listing Resource Groups.");
Console.WriteLine(ex);
}
}
catch (Exception ex)
{
Console.WriteLine($" [ERROR]: Exception during authentication.");
Console.WriteLine(ex);
}
}
Console.WriteLine("Cleanup finished ");
}
public static IAuthenticated Authenticate(string authFile)
{
AzureCredentials credentials = SdkContext.AzureCredentialsFactory.FromFile(authFile);
return(Authenticate(credentials));
}
/// <summary>
/// Returns Azure Credentials
/// </summary>
/// <returns></returns>
public AzureCredentials GetAzureCrendentials()
{
AzureCredentials credentials = SdkContext.AzureCredentialsFactory.FromServicePrincipal(_appSettings.Clientid, _appSettings.Clientsecret, _appSettings.TenantId, AzureEnvironment.AzureGlobalCloud);
return(credentials);
}
public static IAzure GetAzManagementClient(AzureCredentials credentials)
{
return(Azure.Configure().WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic).Authenticate(credentials).WithDefaultSubscription());
}
private static AzureMonitorClient CreateAzureMonitorClient(AzureMetadata azureMetadata, AzureCredentials azureCredentials, ILogger logger)
{
var azureMonitorClient = new AzureMonitorClient(azureMetadata.TenantId, azureMetadata.SubscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, logger);
return(azureMonitorClient);
}
public IBuildable WithCredentials(AzureCredentials credentials)
{
this.credentials = credentials;
return(this);
}
private static async Task Run()
{
string subscriptionIds = ConfigurationManager.AppSettings["subscriptionIds"];
string ownerTagName = ConfigurationManager.AppSettings["ownerTagName"];
string startDate = DateTime.Now.ToUniversalTime().AddDays(-30).ToString("yyyy-MM-ddTHH:mm:ss.fffZ");
string endDate = DateTime.Now.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ");
string clientId = ConfigurationManager.AppSettings["clientId"];
string clientSecret = ConfigurationManager.AppSettings["clientSecret"];
string tenantId = ConfigurationManager.AppSettings["tenantId"];
AzureCredentialsFactory factory = new AzureCredentialsFactory();
AzureCredentials azureCreds = factory.FromServicePrincipal(clientId, clientSecret, tenantId,
AzureEnvironment.AzureGlobalCloud);
Azure.IAuthenticated azure = Azure.Configure().Authenticate(azureCreds);
string body = @"
{
""type"": ""Usage"",
""timeframe"": ""Custom"",
""timePeriod"": {
""from"": """ + startDate + @""",
""to"": """ + endDate + @""",
},
""dataset"": {
""granularity"": ""Daily"",
""aggregation"": {
""totalCost"": {
""name"": ""PreTaxCost"",
""function"": ""Sum""
}
},
""grouping"": [
{
""type"": ""Dimension"",
""name"": ""ResourceGroup""
}
]
}
}
";
string currency = String.Empty;
Dictionary <string, Double> costPerUser = new Dictionary <string, Double>();
Dictionary <string, Double> costPerGroup = new Dictionary <string, Double>();
Dictionary <string, string> groupOwner = new Dictionary <string, string>();
string token = await GetOAuthTokenFromAAD();
foreach (var subscriptionId in subscriptionIds.Split(",", StringSplitOptions.RemoveEmptyEntries))
{
var azureSub = azure.WithSubscription(subscriptionId);
var resourceGroups = azureSub.ResourceGroups.List();
string uri = $"https://management.azure.com/subscriptions/{subscriptionId}/providers/Microsoft.CostManagement/query?api-version=2019-01-01";
while (!String.IsNullOrEmpty(uri))
{
QueryResult result = null;
int costIndex = -1;
int currencyIndex = -1;
int groupIndex = -1;
var request = HttpWebRequest.CreateHttp(uri);
request.ContentType = "application/json";
request.Method = "POST";
request.Headers.Add("Authorization", $"Bearer {token}");
try
{
using (StreamWriter writer = new StreamWriter(request.GetRequestStream()))
{
writer.Write(body);
writer.Flush();
}
var response = await request.GetResponseAsync();
var responseString = String.Empty;
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
{
responseString = reader.ReadToEnd();
}
result = JsonConvert.DeserializeObject <QueryResult>(responseString);
uri = result.properties.nextLink;
costIndex = GetColumnIndex(result.properties.columns, "PreTaxCost");
currencyIndex = GetColumnIndex(result.properties.columns, "Currency");
groupIndex = GetColumnIndex(result.properties.columns, "ResourceGroup");
if (costIndex < 0)
{
Console.WriteLine($"Could not find cost index for subscription {subscriptionId}");
continue;
}
}
catch (WebException wex)
{
string errorMessage = string.Empty;
if (wex.Response != null)
{
using (StreamReader reader = new StreamReader(wex.Response.GetResponseStream()))
{
errorMessage = reader.ReadToEnd();
}
}
Console.WriteLine($"Error while calculating costs for subscription {subscriptionId}: {wex} ({errorMessage})");
}
catch (Exception ex)
{
Console.WriteLine($"Error while calculating costs for subscription {subscriptionId}: {ex}");
}
if (result != null)
{
foreach (var group in resourceGroups)
{
var resourceGroupOwner = OWNER_UNKNOWN;
var defaultKeyValuePair = default(KeyValuePair <String, String>);
var ownerTag = defaultKeyValuePair;
if (group.Tags != null)
{
ownerTag = group.Tags.Where(tag => tag.Key.Equals(ownerTagName, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault();
}
if (!ownerTag.Equals(defaultKeyValuePair))
{
resourceGroupOwner = ownerTag.Value;
}
//Console.WriteLine($"Calculating costs for resource group {group.Name} in subscription {subscriptionId} which belongs to {resourceGroupOwner}");
string keyNameGroup = $"{subscriptionId}/{group.Name}";
if (!costPerUser.ContainsKey(resourceGroupOwner))
{
costPerUser.Add(resourceGroupOwner, 0);
}
if (!costPerGroup.ContainsKey(keyNameGroup))
{
costPerGroup.Add(keyNameGroup, 0);
}
if (!groupOwner.ContainsKey(keyNameGroup))
{
groupOwner.Add(keyNameGroup, resourceGroupOwner);
}
var groupRows = result.properties.rows.Where(rTemp => rTemp[groupIndex].ToString().Equals(group.Name,
StringComparison.InvariantCultureIgnoreCase)).ToArray();
foreach (var row in groupRows)
{
costPerUser[resourceGroupOwner] += (Double)row[costIndex];
costPerGroup[keyNameGroup] += (Double)row[costIndex];
var currencyOfRow = (string)row[currencyIndex];
if (String.IsNullOrEmpty(currency))
{
currency = currencyOfRow;
}
else if (!currency.Equals(currencyOfRow))
{
throw new Exception("There are different currencies");
}
}
}
}
}
Console.WriteLine($"##########################################");
Console.WriteLine($"Cost between {startDate} and {endDate} per resource group for unknown owner");
Console.WriteLine($"##########################################");
var subscriptionRgUnknown = costPerGroup.Where(temp => temp.Key.Split("/")[0].
Equals(subscriptionId, StringComparison.InvariantCultureIgnoreCase));
foreach (KeyValuePair <string, double> costEntry in subscriptionRgUnknown.OrderByDescending(temp => temp.Value))
{
if (groupOwner[costEntry.Key].Equals(OWNER_UNKNOWN, StringComparison.InvariantCultureIgnoreCase))
{
Console.WriteLine($"{costEntry.Key}: {currency} {costEntry.Value}");
}
}
}
Console.WriteLine($"##########################################");
Console.WriteLine($"Cost between {startDate} and {endDate} per user");
Console.WriteLine($"##########################################");
foreach (KeyValuePair <string, double> costEntry in costPerUser.OrderByDescending(temp => temp.Value))
{
Console.WriteLine($"{costEntry.Key}: {currency} {costEntry.Value}");
}
Console.WriteLine($"##########################################");
Console.WriteLine($"Cost between {startDate} and {endDate} per resource group");
Console.WriteLine($"##########################################");
foreach (KeyValuePair <string, double> costEntry in costPerGroup.OrderByDescending(temp => temp.Value))
{
Console.WriteLine($"{costEntry.Key}: {currency} {costEntry.Value} (owner: {groupOwner[costEntry.Key]})");
}
}
public static async Task Run([QueueTrigger("create-items")] TrialResource myQueueItem, ILogger log)
{
log.LogInformation("CreateResource QueueTrigger function processed a request.");
var subscriptionId = myQueueItem.SubscriptionId;;
AzureServiceTokenProvider azureServiceTokenProvider = new AzureServiceTokenProvider();
var tenantId = myQueueItem.TenantId;
var tokenCredentials = new TokenCredentials(await azureServiceTokenProvider.GetAccessTokenAsync("https://management.azure.com/").ConfigureAwait(false));
var azureCredentials = new AzureCredentials(
tokenCredentials,
tokenCredentials,
tenantId,
AzureEnvironment.FromName(myQueueItem.AzureEnvironment));
var client = RestClient
.Configure()
.WithEnvironment(AzureEnvironment.FromName(myQueueItem.AzureEnvironment))
.WithLogLevel(myQueueItem.DeploymentLoggingLevel.ToEnum <HttpLoggingDelegatingHandler.Level>())
.WithCredentials(azureCredentials)
.Build();
var azure = Azure
.Authenticate(client, tenantId)
.WithSubscription(subscriptionId);
// var resourceManagementClient = new ResourceManagementClient(client);
string rgName = myQueueItem.ResourceGroupName; //SdkContext.RandomResourceName(myQueueItem.ResourceGroupName, 24);
string deploymentName = myQueueItem.AppServiceName;
try
{
//var templateJson = File.ReadAllText(System.IO.Path.Combine(context.FunctionDirectory, "..\\FreeFunctionARMTemplate.json"));
//=============================================================
// Create resource group.
Console.WriteLine("Creating a resource group with name: " + rgName);
await azure.ResourceGroups.Define(rgName)
.WithRegion(myQueueItem.Region.ToEnum <Region>())
.CreateAsync();
Console.WriteLine("Created a resource group with name: " + rgName);
//=============================================================
// Create a deployment for an Azure App Service via an ARM template.
Console.WriteLine("Starting a deployment for an Azure App Service: " + deploymentName);
await azure.Deployments.Define(deploymentName)
.WithExistingResourceGroup(rgName)
.WithTemplateLink("", null)
//.WithParameters(new Dictionary<string, Dictionary<string, object>>{
// { "hostingPlanName", new Dictionary<string, object>{{"value",deploymentName}}},
// { "skuName", new Dictionary<string, object>{{"value", "B1" }}},
// { "skuCapacity", new Dictionary<string, object>{{"value",1}}},
// { "webSiteName", new Dictionary<string, object>{{"value", deploymentName } }}
// })
//.WithParameters(new Dictionary<string, Dictionary<string, object>>{
// { "msdeployPackageUrl", new Dictionary<string, object>{{"value", "https://tryappservicetemplates.blob.core.windows.net/zipped/Default/Express.zip" } }},
// { "appServiceName", new Dictionary<string, object>{{"value", deploymentName } }}
// })
.WithParametersLink("", null)
.WithMode(myQueueItem.DeploymentMode.ToEnum <DeploymentMode>())
.CreateAsync();
Console.WriteLine("Started a deployment for an Azure App Service: " + deploymentName);
var deployment = await azure.Deployments.GetByResourceGroupAsync(rgName, deploymentName);
Console.WriteLine("Current deployment status : " + deployment.ProvisioningState);
var tries = 180;
while (!(StringComparer.OrdinalIgnoreCase.Equals(deployment.ProvisioningState, "Succeeded") ||
StringComparer.OrdinalIgnoreCase.Equals(deployment.ProvisioningState, "Failed") ||
StringComparer.OrdinalIgnoreCase.Equals(deployment.ProvisioningState, "Cancelled")) && tries-- > 0)
{
SdkContext.DelayProvider.Delay(10000);
deployment = await azure.Deployments.GetByResourceGroupAsync(rgName, deploymentName);
Console.WriteLine("Current deployment status : " + deployment.ProvisioningState);
}
if (StringComparer.OrdinalIgnoreCase.Equals(deployment.ProvisioningState, "Succeeded"))
{
var res = deployment.Outputs;
var sitesDeployed = deployment.Dependencies.Where(a => StringComparer.OrdinalIgnoreCase.Equals(a.ResourceType, "Microsoft.Web/sites"));
if (sitesDeployed != null)
{
var siteList = new List <IWebApp>();
foreach (var site in sitesDeployed)
{
siteList.Add(await azure.WebApps.GetByIdAsync(site.Id));
}
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
finally
{
try
{
Console.WriteLine("Deleting Resource Group: " + rgName);
azure.ResourceGroups.DeleteByName(rgName);
Console.WriteLine("Deleted Resource Group: " + rgName);
}
catch (NullReferenceException)
{
Console.WriteLine("Did not create any resources in Azure. No clean up is necessary");
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
string name = myQueueItem.AppServiceName;
}
public static void Main(string[] args)
{
try
{
//=================================================================
// Authenticate
var credentials = AzureCredentials.FromFile(Environment.GetEnvironmentVariable("AZURE_AUTH_LOCATION"));
var azure = Azure
.Configure()
.WithLogLevel(HttpLoggingDelegatingHandler.Level.BASIC)
.Authenticate(credentials)
.WithDefaultSubscription();
// Print selected subscription
Console.WriteLine("Selected subscription: " + azure.SubscriptionId);
try
{
//=============================================================
// Create a virtual network with a frontend and a backend subnets
Console.WriteLine("Creating virtual network with a frontend and a backend subnets...");
var network = azure.Networks.Define(vnetName)
.WithRegion(Region.US_EAST)
.WithNewResourceGroup(rgName)
.WithAddressSpace("172.16.0.0/16")
.DefineSubnet("Front-end")
.WithAddressPrefix("172.16.1.0/24")
.Attach()
.DefineSubnet("Back-end")
.WithAddressPrefix("172.16.3.0/24")
.Attach()
.Create();
Console.WriteLine("Created a virtual network");
// Print the virtual network details
Utilities.PrintVirtualNetwork(network);
//=============================================================
// Create a public IP address
Console.WriteLine("Creating a public IP address...");
var publicIpAddress = azure.PublicIpAddresses.Define(publicIpName1)
.WithRegion(Region.US_EAST)
.WithExistingResourceGroup(rgName)
.WithLeafDomainLabel(publicIpName1)
.Create();
Console.WriteLine("Created a public IP address");
// Print the virtual network details
Utilities.PrintIpAddress(publicIpAddress);
//=============================================================
// Create an Internet facing load balancer
// Create a frontend IP address
// Two backend address pools which contain network interfaces for the virtual
// machines to receive HTTP and HTTPS network traffic from the load balancer
// Two load balancing rules for HTTP and HTTPS to map public ports on the load
// balancer to ports in the backend address pool
// Two probes which contain HTTP and HTTPS health probes used to check availability
// of virtual machines in the backend address pool
// Two inbound NAT rules which contain rules that map a public port on the load
// balancer to a port for a specific virtual machine in the backend address pool
// - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23
Console.WriteLine("Creating a Internet facing load balancer with ...");
Console.WriteLine("- A frontend IP address");
Console.WriteLine("- Two backend address pools which contain network interfaces for the virtual\n"
+ " machines to receive HTTP and HTTPS network traffic from the load balancer");
Console.WriteLine("- Two load balancing rules for HTTP and HTTPS to map public ports on the load\n"
+ " balancer to ports in the backend address pool");
Console.WriteLine("- Two probes which contain HTTP and HTTPS health probes used to check availability\n"
+ " of virtual machines in the backend address pool");
Console.WriteLine("- Two inbound NAT rules which contain rules that map a public port on the load\n"
+ " balancer to a port for a specific virtual machine in the backend address pool\n"
+ " - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23");
var loadBalancer1 = azure.LoadBalancers.Define(loadBalancerName1)
.WithRegion(Region.US_EAST)
.WithExistingResourceGroup(rgName)
.DefinePublicFrontend(frontendName)
.WithExistingPublicIpAddress(publicIpAddress)
.Attach()
// Add two backend one per rule
.DefineBackend(backendPoolName1)
.Attach()
.DefineBackend(backendPoolName2)
.Attach()
// Add two probes one per rule
.DefineHttpProbe(httpProbe)
.WithRequestPath("/")
.WithPort(80)
.Attach()
.DefineHttpProbe(httpsProbe)
.WithRequestPath("/")
.WithPort(443)
.Attach()
// Add two rules that uses above backend and probe
.DefineLoadBalancingRule(httpLoadBalancingRule)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(80)
.WithProbe(httpProbe)
.WithBackend(backendPoolName1)
.Attach()
.DefineLoadBalancingRule(httpsLoadBalancingRule)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(443)
.WithProbe(httpsProbe)
.WithBackend(backendPoolName2)
.Attach()
// Add two nat pools to enable direct VM connectivity for
// SSH to port 22 and TELNET to port 23
.DefineInboundNatRule(natRule5000to22forVM1)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(5000)
.WithBackendPort(22)
.Attach()
.DefineInboundNatRule(natRule5001to23forVM1)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(5001)
.WithBackendPort(23)
.Attach()
.DefineInboundNatRule(natRule5002to22forVM2)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(5002)
.WithBackendPort(22)
.Attach()
.DefineInboundNatRule(natRule5003to23forVM2)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(5003)
.WithBackendPort(23)
.Attach()
.Create();
// Print load balancer details
Console.WriteLine("Created a load balancer");
Utilities.PrintLoadBalancer(loadBalancer1);
//=============================================================
// Create two network interfaces in the frontend subnet
// associate network interfaces to NAT rules, backend pools
Console.WriteLine("Creating two network interfaces in the frontend subnet ...");
Console.WriteLine("- And associating network interfaces to backend pools and NAT rules");
var networkInterfaceCreatables = new List <ICreatable <INetworkInterface> >();
ICreatable <INetworkInterface> networkInterface1Creatable;
ICreatable <INetworkInterface> networkInterface2Creatable;
networkInterface1Creatable = azure.NetworkInterfaces
.Define(networkInterfaceName1)
.WithRegion(Region.US_EAST)
.WithNewResourceGroup(rgName)
.WithExistingPrimaryNetwork(network)
.WithSubnet("Front-end")
.WithPrimaryPrivateIpAddressDynamic()
.WithExistingLoadBalancerBackend(loadBalancer1, backendPoolName1)
.WithExistingLoadBalancerBackend(loadBalancer1, backendPoolName2)
.WithExistingLoadBalancerInboundNatRule(loadBalancer1, natRule5000to22forVM1)
.WithExistingLoadBalancerInboundNatRule(loadBalancer1, natRule5001to23forVM1);
networkInterfaceCreatables.Add(networkInterface1Creatable);
networkInterface2Creatable = azure.NetworkInterfaces
.Define(networkInterfaceName2)
.WithRegion(Region.US_EAST)
.WithNewResourceGroup(rgName)
.WithExistingPrimaryNetwork(network)
.WithSubnet("Front-end")
.WithPrimaryPrivateIpAddressDynamic()
.WithExistingLoadBalancerBackend(loadBalancer1, backendPoolName1)
.WithExistingLoadBalancerBackend(loadBalancer1, backendPoolName2)
.WithExistingLoadBalancerInboundNatRule(loadBalancer1, natRule5002to22forVM2)
.WithExistingLoadBalancerInboundNatRule(loadBalancer1, natRule5003to23forVM2);
networkInterfaceCreatables.Add(networkInterface2Creatable);
var networkInterfaces1 = azure.NetworkInterfaces.Create(networkInterfaceCreatables.ToArray());
// Print network interface details
Console.WriteLine("Created two network interfaces");
Console.WriteLine("Network Interface ONE -");
Utilities.PrintNetworkInterface(networkInterfaces1.ElementAt(0));
Console.WriteLine();
Console.WriteLine("Network Interface TWO -");
Utilities.PrintNetworkInterface(networkInterfaces1.ElementAt(1));
//=============================================================
// Create an availability set
Console.WriteLine("Creating an availability set ...");
var availSet1 = azure.AvailabilitySets.Define(availSetName)
.WithRegion(Region.US_EAST)
.WithNewResourceGroup(rgName)
.WithFaultDomainCount(2)
.WithUpdateDomainCount(4)
.Create();
Console.WriteLine("Created first availability set: " + availSet1.Id);
Utilities.PrintAvailabilitySet(availSet1);
//=============================================================
// Create two virtual machines and assign network interfaces
Console.WriteLine("Creating two virtual machines in the frontend subnet ...");
Console.WriteLine("- And assigning network interfaces");
var virtualMachineCreatables1 = new List <ICreatable <IVirtualMachine> >();
ICreatable <IVirtualMachine> virtualMachine1Creatable;
ICreatable <IVirtualMachine> virtualMachine2Creatable;
virtualMachine1Creatable = azure.VirtualMachines
.Define(vmName1)
.WithRegion(Region.US_EAST)
.WithExistingResourceGroup(rgName)
.WithExistingPrimaryNetworkInterface(networkInterfaces1.ElementAt(0))
.WithPopularLinuxImage(KnownLinuxVirtualMachineImage.UBUNTU_SERVER_16_04_LTS)
.WithRootUserName(userName)
.WithSsh(sshKey)
.WithSize(VirtualMachineSizeTypes.StandardD3V2)
.WithExistingAvailabilitySet(availSet1);
virtualMachineCreatables1.Add(virtualMachine1Creatable);
virtualMachine2Creatable = azure.VirtualMachines
.Define(vmName2)
.WithRegion(Region.US_EAST)
.WithExistingResourceGroup(rgName)
.WithExistingPrimaryNetworkInterface(networkInterfaces1.ElementAt(1))
.WithPopularLinuxImage(KnownLinuxVirtualMachineImage.UBUNTU_SERVER_16_04_LTS)
.WithRootUserName(userName)
.WithSsh(sshKey)
.WithSize(VirtualMachineSizeTypes.StandardD3V2)
.WithExistingAvailabilitySet(availSet1);
virtualMachineCreatables1.Add(virtualMachine2Creatable);
var t1 = DateTime.UtcNow;
var virtualMachines = azure.VirtualMachines.Create(virtualMachineCreatables1.ToArray());
var t2 = DateTime.UtcNow;
Console.WriteLine($"Created 2 Linux VMs: (took {(t2 - t1).TotalSeconds} seconds) ");
Console.WriteLine();
// Print virtual machine details
Console.WriteLine("Virtual Machine ONE -");
Utilities.PrintVirtualMachine(virtualMachines.ElementAt(0));
Console.WriteLine();
Console.WriteLine("Virtual Machine TWO - ");
Utilities.PrintVirtualMachine(virtualMachines.ElementAt(1));
//=============================================================
// Update a load balancer
// configure TCP idle timeout to 15 minutes
Console.WriteLine("Updating the load balancer ...");
loadBalancer1.Update()
.UpdateLoadBalancingRule(httpLoadBalancingRule)
.WithIdleTimeoutInMinutes(15)
.Parent()
.UpdateLoadBalancingRule(httpsLoadBalancingRule)
.WithIdleTimeoutInMinutes(15)
.Parent()
.Apply();
Console.WriteLine("Update the load balancer with a TCP idle timeout to 15 minutes");
//=============================================================
// Create another public IP address
Console.WriteLine("Creating another public IP address...");
var publicIpAddress2 = azure.PublicIpAddresses.Define(publicIpName2)
.WithRegion(Region.US_EAST)
.WithExistingResourceGroup(rgName)
.WithLeafDomainLabel(publicIpName2)
.Create();
Console.WriteLine("Created another public IP address");
// Print the virtual network details
Utilities.PrintIpAddress(publicIpAddress2);
//=============================================================
// Create another Internet facing load balancer
// Create a frontend IP address
// Two backend address pools which contain network interfaces for the virtual
// machines to receive HTTP and HTTPS network traffic from the load balancer
// Two load balancing rules for HTTP and HTTPS to map public ports on the load
// balancer to ports in the backend address pool
// Two probes which contain HTTP and HTTPS health probes used to check availability
// of virtual machines in the backend address pool
// Two inbound NAT rules which contain rules that map a public port on the load
// balancer to a port for a specific virtual machine in the backend address pool
// - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23
Console.WriteLine("Creating another Internet facing load balancer with ...");
Console.WriteLine("- A frontend IP address");
Console.WriteLine("- Two backend address pools which contain network interfaces for the virtual\n"
+ " machines to receive HTTP and HTTPS network traffic from the load balancer");
Console.WriteLine("- Two load balancing rules for HTTP and HTTPS to map public ports on the load\n"
+ " balancer to ports in the backend address pool");
Console.WriteLine("- Two probes which contain HTTP and HTTPS health probes used to check availability\n"
+ " of virtual machines in the backend address pool");
Console.WriteLine("- Two inbound NAT rules which contain rules that map a public port on the load\n"
+ " balancer to a port for a specific virtual machine in the backend address pool\n"
+ " - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23");
var loadBalancer2 = azure.LoadBalancers.Define(loadBalancerName2)
.WithRegion(Region.US_EAST)
.WithExistingResourceGroup(rgName)
.DefinePublicFrontend(frontendName)
.WithExistingPublicIpAddress(publicIpAddress2)
.Attach()
// Add two backend one per rule
.DefineBackend(backendPoolName1)
.Attach()
.DefineBackend(backendPoolName2)
.Attach()
// Add two probes one per rule
.DefineHttpProbe(httpProbe)
.WithRequestPath("/")
.WithPort(80)
.Attach()
.DefineHttpProbe(httpsProbe)
.WithRequestPath("/")
.WithPort(443)
.Attach()
// Add two rules that uses above backend and probe
.DefineLoadBalancingRule(httpLoadBalancingRule)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(80)
.WithProbe(httpProbe)
.WithBackend(backendPoolName1)
.Attach()
.DefineLoadBalancingRule(httpsLoadBalancingRule)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(443)
.WithProbe(httpsProbe)
.WithBackend(backendPoolName2)
.Attach()
// Add two nat pools to enable direct VM connectivity for
// SSH to port 22 and TELNET to port 23
.DefineInboundNatRule(natRule5000to22forVM1)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(5000)
.WithBackendPort(22)
.Attach()
.DefineInboundNatRule(natRule5001to23forVM1)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(5001)
.WithBackendPort(23)
.Attach()
.DefineInboundNatRule(natRule5002to22forVM2)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(5002)
.WithBackendPort(22)
.Attach()
.DefineInboundNatRule(natRule5003to23forVM2)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(5003)
.WithBackendPort(23)
.Attach()
.Create();
// Print load balancer details
Console.WriteLine("Created another load balancer");
Utilities.PrintLoadBalancer(loadBalancer2);
//=============================================================
// List load balancers
var loadBalancers = azure.LoadBalancers.List();
Console.WriteLine("Walking through the list of load balancers");
foreach (var loadBalancer in loadBalancers)
{
Utilities.PrintLoadBalancer(loadBalancer);
}
//=============================================================
// Remove a load balancer
Console.WriteLine("Deleting load balancer " + loadBalancerName2
+ "(" + loadBalancer2.Id + ")");
azure.LoadBalancers.Delete(loadBalancer2.Id);
Console.WriteLine("Deleted load balancer" + loadBalancerName2);
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
finally
{
try
{
Console.WriteLine("Deleting Resource Group: " + rgName);
azure.ResourceGroups.Delete(rgName);
Console.WriteLine("Deleted Resource Group: " + rgName);
}
catch (NullReferenceException)
{
Console.WriteLine("Did not create any resources in Azure. No clean up is necessary");
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
/// <summary>
/// Initialize Resources on Azure. Create a resource group, a storage, a container inside the storage,
/// and update template.json and parameter.json onto it. They will be used for deployment.
/// </summary>
public static void InitializeResources()
{
Console.WriteLine("Initialize Resources...");
// Getting the Azure AD application credentials
AzureCredentials credentials = SdkContext.AzureCredentialsFactory.FromFile("../../azureauth.properties");
// Logging to my Azure AD
azure = Azure.Configure().WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic).Authenticate(credentials).WithDefaultSubscription();
// Creating a new ResourceGroup with the name of rgName
spin.setMessage("Creating a resource Group...");
spin.Start();
var resourceGroup = azure.ResourceGroups.Define(rgName).WithRegion(location).Create();
spin.Stop();
Console.WriteLine($"Resource Group {rgName} created");
// Creating the storage linked to the resource Group
spin.setMessage($"Creating the storage {accountStorageName}...");
spin.Start();
var storage = azure.StorageAccounts.Define(accountStorageName).WithRegion(location).WithExistingResourceGroup(resourceGroup).Create();
var storageKeys = storage.GetKeys();
string storageConnectionString = "DefaultEndpointsProtocol=https;"
+ "AccountName=" + storage.Name
+ ";AccountKey=" + storageKeys[0].Value
+ ";EndpointSuffix=core.windows.net";
//Connexion to our newly created storage account and setting up our service blob client
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient serviceClient = cloudStorageAccount.CreateCloudBlobClient();
// Creating our container
//Console.WriteLine("Creating container...");
CloudBlobContainer container = serviceClient.GetContainerReference("templates");
container.CreateIfNotExistsAsync().GetAwaiter().GetResult();
BlobContainerPermissions containerPermissions = new BlobContainerPermissions()
{
PublicAccess = BlobContainerPublicAccessType.Container
};
container.SetPermissionsAsync(containerPermissions).GetAwaiter().GetResult();
Console.WriteLine($"Storage {accountStorageName} created");
spin.Stop();
//Uploading to the container our template file
spin.setMessage("Uploading template file...");
spin.Start();
CloudBlockBlob templateBlob = container.GetBlockBlobReference("template.json");
templateBlob.UploadFromFileAsync("..\\..\\Templates\\template.json").GetAwaiter().GetResult();
//uploading to the container our parameter file
Console.WriteLine("Uploading parameter file...");
CloudBlockBlob paramBlob = container.GetBlockBlobReference("parameters.json");
paramBlob.UploadFromFileAsync("..\\..\\Templates\\parameters.json").GetAwaiter().GetResult();
spin.Stop();
}
public static void Main(string[] args)
{
try
{
var rgName = ResourceNamer.RandomResourceName("rgRSMA", 24);
var rgName2 = ResourceNamer.RandomResourceName("rgRSMA", 24);
var resourceTagName = ResourceNamer.RandomResourceName("rgRSTN", 24);
var resourceTagValue = ResourceNamer.RandomResourceName("rgRSTV", 24);
try
{
//=================================================================
// Authenticate
AzureCredentials credentials = AzureCredentials.FromFile(Environment.GetEnvironmentVariable("AZURE_AUTH_LOCATION"));
var azure = Azure
.Configure()
.WithLogLevel(HttpLoggingDelegatingHandler.Level.BASIC)
.Authenticate(credentials)
.WithDefaultSubscription();
try
{
//=============================================================
// Create resource group.
Console.WriteLine("Creating a resource group with name: " + rgName);
var resourceGroup = azure.ResourceGroups
.Define(rgName)
.WithRegion(Region.US_WEST)
.Create();
Console.WriteLine("Created a resource group with name: " + rgName);
//=============================================================
// Update the resource group.
Console.WriteLine("Updating the resource group with name: " + rgName);
resourceGroup.Update()
.WithTag(resourceTagName, resourceTagValue)
.Apply();
Console.WriteLine("Updated the resource group with name: " + rgName);
//=============================================================
// Create another resource group.
Console.WriteLine("Creating another resource group with name: " + rgName2);
var resourceGroup2 = azure.ResourceGroups
.Define(rgName2)
.WithRegion(Region.US_WEST)
.Create();
Console.WriteLine("Created another resource group with name: " + rgName2);
//=============================================================
// List resource groups.
Console.WriteLine("Listing all resource groups");
foreach (var rGroup in azure.ResourceGroups.List())
{
Console.WriteLine("Resource group: " + rGroup.Name);
}
//=============================================================
// Delete a resource group.
Console.WriteLine("Deleting resource group: " + rgName2);
azure.ResourceGroups.Delete(rgName2);
Console.WriteLine("Deleted resource group: " + rgName2);
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
finally
{
try
{
Console.WriteLine("Deleting Resource Group: " + rgName);
azure.ResourceGroups.Delete(rgName);
Console.WriteLine("Deleted Resource Group: " + rgName);
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
public static void Main(string[] args)
{
try
{
//=============================================================
// Authenticate
var credentials = AzureCredentials.FromFile(Environment.GetEnvironmentVariable("AZURE_AUTH_LOCATION"));
var azure = Azure
.Configure()
.WithLogLevel(HttpLoggingDelegatingHandler.Level.BASIC)
.Authenticate(credentials)
.WithDefaultSubscription();
// Print selected subscription
Console.WriteLine("Selected subscription: " + azure.SubscriptionId);
try
{
//=============================================================
// Create a virtual network with a frontend subnet
Console.WriteLine("Creating virtual network with a frontend subnet ...");
var network = azure.Networks.Define(vnetName)
.WithRegion(Region.US_EAST)
.WithNewResourceGroup(rgName)
.WithAddressSpace("172.16.0.0/16")
.DefineSubnet("Front-end")
.WithAddressPrefix("172.16.1.0/24")
.Attach()
.Create();
Console.WriteLine("Created a virtual network");
// Print the virtual network details
Utilities.PrintVirtualNetwork(network);
//=============================================================
// Create a public IP address
Console.WriteLine("Creating a public IP address...");
var publicIpAddress = azure.PublicIpAddresses.Define(publicIpName)
.WithRegion(Region.US_EAST)
.WithExistingResourceGroup(rgName)
.WithLeafDomainLabel(publicIpName)
.Create();
Console.WriteLine("Created a public IP address");
// Print the IPAddress details
Utilities.PrintIpAddress(publicIpAddress);
//=============================================================
// Create an Internet facing load balancer with
// One frontend IP address
// Two backend address pools which contain network interfaces for the virtual
// machines to receive HTTP and HTTPS network traffic from the load balancer
// Two load balancing rules for HTTP and HTTPS to map public ports on the load
// balancer to ports in the backend address pool
// Two probes which contain HTTP and HTTPS health probes used to check availability
// of virtual machines in the backend address pool
// Three inbound NAT rules which contain rules that map a public port on the load
// balancer to a port for a specific virtual machine in the backend address pool
// - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23
Console.WriteLine("Creating a Internet facing load balancer with ...");
Console.WriteLine("- A frontend IP address");
Console.WriteLine("- Two backend address pools which contain network interfaces for the virtual\n"
+ " machines to receive HTTP and HTTPS network traffic from the load balancer");
Console.WriteLine("- Two load balancing rules for HTTP and HTTPS to map public ports on the load\n"
+ " balancer to ports in the backend address pool");
Console.WriteLine("- Two probes which contain HTTP and HTTPS health probes used to check availability\n"
+ " of virtual machines in the backend address pool");
Console.WriteLine("- Two inbound NAT rules which contain rules that map a public port on the load\n"
+ " balancer to a port for a specific virtual machine in the backend address pool\n"
+ " - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23");
var loadBalancer1 = azure.LoadBalancers.Define(loadBalancerName1)
.WithRegion(Region.US_EAST)
.WithExistingResourceGroup(rgName)
.DefinePublicFrontend(frontendName)
.WithExistingPublicIpAddress(publicIpAddress)
.Attach()
// Add two backend one per rule
.DefineBackend(backendPoolName1)
.Attach()
.DefineBackend(backendPoolName2)
.Attach()
// Add two probes one per rule
.DefineHttpProbe(httpProbe)
.WithRequestPath("/")
.WithPort(80)
.Attach()
.DefineHttpProbe(httpsProbe)
.WithRequestPath("/")
.WithPort(443)
.Attach()
// Add two rules that uses above backend and probe
.DefineLoadBalancingRule(httpLoadBalancingRule)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(80)
.WithProbe(httpProbe)
.WithBackend(backendPoolName1)
.Attach()
.DefineLoadBalancingRule(httpsLoadBalancingRule)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPort(443)
.WithProbe(httpsProbe)
.WithBackend(backendPoolName2)
.Attach()
// Add nat pools to enable direct VM connectivity for
// SSH to port 22 and TELNET to port 23
.DefineInboundNatPool(natPool50XXto22)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPortRange(5000, 5099)
.WithBackendPort(22)
.Attach()
.DefineInboundNatPool(natPool60XXto23)
.WithProtocol(TransportProtocol.Tcp)
.WithFrontend(frontendName)
.WithFrontendPortRange(6000, 6099)
.WithBackendPort(23)
.Attach()
.Create();
// Print load balancer details
Console.WriteLine("Created a load balancer");
Utilities.PrintLoadBalancer(loadBalancer1);
//=============================================================
// Create a virtual machine scale set with three virtual machines
// And, install Apache Web servers on them
Console.WriteLine("Creating virtual machine scale set with three virtual machines"
+ " in the frontend subnet ...");
var t1 = DateTime.UtcNow;
var fileUris = new List <string>();
fileUris.Add(apacheInstallScript);
var virtualMachineScaleSet = azure.VirtualMachineScaleSets
.Define(vmssName)
.WithRegion(Region.US_EAST)
.WithExistingResourceGroup(rgName)
.WithSku(VirtualMachineScaleSetSkuTypes.STANDARD_D3_V2)
.WithExistingPrimaryNetworkSubnet(network, "Front-end")
.WithPrimaryInternetFacingLoadBalancer(loadBalancer1)
.WithPrimaryInternetFacingLoadBalancerBackends(backendPoolName1, backendPoolName2)
.WithPrimaryInternetFacingLoadBalancerInboundNatPools(natPool50XXto22, natPool60XXto23)
.WithoutPrimaryInternalLoadBalancer()
.WithPopularLinuxImage(KnownLinuxVirtualMachineImage.UBUNTU_SERVER_16_04_LTS)
.WithRootUserName(userName)
.WithSsh(sshKey)
.WithNewStorageAccount(storageAccountName1)
.WithNewStorageAccount(storageAccountName2)
.WithNewStorageAccount(storageAccountName3)
.WithCapacity(3)
// Use a VM extension to install Apache Web servers
.DefineNewExtension("CustomScriptForLinux")
.WithPublisher("Microsoft.OSTCExtensions")
.WithType("CustomScriptForLinux")
.WithVersion("1.4")
.WithMinorVersionAutoUpgrade()
.WithPublicSetting("fileUris", fileUris)
.WithPublicSetting("commandToExecute", installCommand)
.Attach()
.Create();
var t2 = DateTime.UtcNow;
Console.WriteLine("Created a virtual machine scale set with "
+ "3 Linux VMs & Apache Web servers on them: (took "
+ ((t2 - t1).TotalSeconds) + " seconds) ");
Console.WriteLine();
// Print virtual machine scale set details
// Utilities.Print(virtualMachineScaleSet);
//=============================================================
// Stop the virtual machine scale set
Console.WriteLine("Stopping virtual machine scale set ...");
virtualMachineScaleSet.PowerOff();
Console.WriteLine("Stopped virtual machine scale set");
//=============================================================
// Start the virtual machine scale set
Console.WriteLine("Starting virtual machine scale set ...");
virtualMachineScaleSet.Start();
Console.WriteLine("Started virtual machine scale set");
//=============================================================
// Update the virtual machine scale set
// - double the no. of virtual machines
Console.WriteLine("Updating virtual machine scale set "
+ "- double the no. of virtual machines ...");
virtualMachineScaleSet.Update()
.WithCapacity(6)
.Apply();
Console.WriteLine("Doubled the no. of virtual machines in "
+ "the virtual machine scale set");
//=============================================================
// re-start virtual machine scale set
Console.WriteLine("re-starting virtual machine scale set ...");
virtualMachineScaleSet.Restart();
Console.WriteLine("re-started virtual machine scale set");
}
catch (Exception f)
{
Console.WriteLine(f);
}
finally
{
try
{
Console.WriteLine("Deleting Resource Group: " + rgName);
azure.ResourceGroups.Delete(rgName);
Console.WriteLine("Deleted Resource Group: " + rgName);
}
catch (NullReferenceException npe)
{
Console.WriteLine("Did not create any resources in Azure. No clean up is necessary");
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
public ServiceBusQueueScraper(AzureMetadata azureMetadata, AzureCredentials azureCredentials, IExceptionTracker exceptionTracker)
: base(azureMetadata, azureCredentials, exceptionTracker)
{
}
public ISqlManager Authenticate(AzureCredentials credentials, string subscriptionId)
{
return(new SqlManager(BuildRestClient(credentials), subscriptionId, credentials.TenantId));
}
public abstract void Initialize(AzureCredentials credentials);
/**
* Azure Compute sample for managing virtual machine from Managed Service Identity (MSI) enabled virtual machine -
* - Create a virtual machine using MSI credentials from MSI enabled VM
* - Delete the virtual machine using MSI credentials from MSI enabled VM.
*
* Automation testing cannot be enabled for this sample since it can be run only from an Azure virtual machine
* with MSI enabled.
*/
public static void Main(string[] args)
{
var rgName = "msi-rg-test";
var region = Region.USWestCentral;
// This sample required to be run from a MSI enabled virtual machine with role
// based contributor access to the resource group with name "msi-rg-test". MSI
// enabled VM can be created using service principal credentials as shown below.
//var credFile = Environment.GetEnvironmentVariable("AZURE_AUTH_LOCATION");
//IAzure azure = Azure.Authenticate(credFile).WithDefaultSubscription();
//var virtualMachine = azure.VirtualMachines
// .Define("<vm-name>")
// .WithRegion(region)
// .WithNewResourceGroup(rgName)
// .WithNewPrimaryNetwork("10.0.0.0/28")
// .WithPrimaryPrivateIPAddressDynamic()
// .WithNewPrimaryPublicIPAddress(pipName)
// .WithPopularLinuxImage(KnownLinuxVirtualMachineImage.UbuntuServer16_04_Lts)
// .WithRootUsername("<user-name>")
// .WithRootPassword("<password>")
// .WithSize(VirtualMachineSizeTypes.StandardDS2V2)
// .WithOSDiskCaching(CachingTypes.ReadWrite)
// .WithManagedServiceIdentity() // Enable MSI
// .WithRoleBasedAccessToCurrentResourceGroup(BuiltInRole.Contributor) // With MSI Role assignment to current resource group
// .Create();
// Specify your subscription Id
string subscriptionId = "<subscription-id>";
var linuxVmName = Utilities.CreateRandomName("VM1");
var userName = "******";
var password = "******";
//=============================================================
// MSI Authenticate
AzureCredentials credentials = new AzureCredentials(new MSILoginInformation {
Port = 50342
}, AzureEnvironment.AzureGlobalCloud);
IAzure azure = Azure.Authenticate(credentials)
.WithSubscription(subscriptionId);
//=============================================================
// Create a Linux VM using MSI credentials
Console.WriteLine("Creating a Linux VM using MSI credentials");
var virtualMachine = azure.VirtualMachines
.Define(linuxVmName)
.WithRegion(region)
.WithExistingResourceGroup(rgName)
.WithNewPrimaryNetwork("10.0.0.0/28")
.WithPrimaryPrivateIPAddressDynamic()
.WithoutPrimaryPublicIPAddress()
.WithPopularLinuxImage(KnownLinuxVirtualMachineImage.UbuntuServer16_04_Lts)
.WithRootUsername(userName)
.WithRootPassword(password)
.WithSize(VirtualMachineSizeTypes.StandardDS2V2)
.WithOSDiskCaching(CachingTypes.ReadWrite)
.Create();
Console.WriteLine("Created virtual machine using MSI credentials");
Utilities.PrintVirtualMachine(virtualMachine);
//=============================================================
// Delete the VM using MSI credentials
Console.WriteLine("Deleting the virtual machine using MSI credentials");
azure.VirtualMachines.DeleteById(virtualMachine.Id);
Console.WriteLine("Deleted virtual machine");
}
private async Task <string> DeployVMScript(string resourceName, string region, string deploymentJson, string deploymentParameters, AzureCredentials creds)
{
Guard.Against.NullOrEmpty(resourceName, nameof(resourceName));
Guard.Against.NullOrEmpty(region, nameof(region));
Guard.Against.NullOrEmpty(deploymentJson, nameof(deploymentJson));
Guard.Against.NullOrEmpty(deploymentParameters, nameof(deploymentParameters));
Guard.Against.Null(creds, nameof(creds));
var azure = AzureClient(creds);
var resourceGroup = await azure.ResourceGroups.Define(resourceName)
.WithRegion(region)
.CreateAsync();
var storageName = SdkContext.RandomResourceName("st", 10);
var storage = await azure.StorageAccounts.Define(storageName)
.WithRegion(region)
.WithExistingResourceGroup(resourceGroup)
.CreateAsync();
var storageKeys = storage.GetKeys();
string storageConnectionString = "DefaultEndpointsProtocol=https;"
+ "AccountName=" + storage.Name
+ ";AccountKey=" + storageKeys[0].Value
+ ";EndpointSuffix=core.windows.net";
var account = CloudStorageAccount.Parse(storageConnectionString);
var serviceClient = account.CreateCloudBlobClient();
var container = serviceClient.GetContainerReference("templates");
await container.CreateIfNotExistsAsync();
var containerPermissions = new BlobContainerPermissions
{
PublicAccess = BlobContainerPublicAccessType.Container
};
await container.SetPermissionsAsync(containerPermissions);
var shellScriptBlob = container.GetBlockBlobReference(_azureSettings.Value.ShellScriptFileName);
await shellScriptBlob.UploadFromFileAsync(shellScriptPath);
string newParametersJson = JsonConvert.DeserializeObject(deploymentParameters).ToString();
newParametersJson = newParametersJson.Replace("[dnsName]", resourceName, StringComparison.CurrentCultureIgnoreCase);
newParametersJson = newParametersJson.Replace("[location]", region, StringComparison.CurrentCultureIgnoreCase);
var newParametersFileName = resourceName + "-parameters" + ".json";
var fileParametersPath = Path.Combine(jsonFilePath + newParametersFileName);
//Write to New File
File.WriteAllText(fileParametersPath, newParametersJson);
//Template write to file
string templateJson = JsonConvert.DeserializeObject(deploymentJson).ToString();
var shellRemotePath = "https://" + storageName + ".blob.core.windows.net/templates/" + _azureSettings.Value.ShellScriptFileName;
templateJson = templateJson.Replace("[shellRemotePath]", shellRemotePath, StringComparison.CurrentCultureIgnoreCase);
var templateJsonFileName = resourceName + "-deploymentJson" + ".json";
var fileTemplatePath = Path.Combine(jsonFilePath + templateJsonFileName);
File.WriteAllText(fileTemplatePath, templateJson);
var templateblob = container.GetBlockBlobReference(templateJsonFileName);
await templateblob.UploadFromFileAsync(fileTemplatePath);
var paramblob = container.GetBlockBlobReference(newParametersFileName);
await paramblob.UploadFromFileAsync(fileParametersPath);
var templatePath = "https://" + storageName + ".blob.core.windows.net/templates/" + templateJsonFileName;
var paramPath = "https://" + storageName + ".blob.core.windows.net/templates/" + newParametersFileName;
var deployment = await azure.Deployments.Define("myDeployment")
.WithExistingResourceGroup(resourceName)
.WithTemplateLink(templatePath, "1.0.0.0")
.WithParametersLink(paramPath, "1.0.0.0")
.WithMode(Microsoft.Azure.Management.ResourceManager.Fluent.Models.DeploymentMode.Incremental)
.CreateAsync();
return(await GetIPAddressForResourceVM(resourceName, azure));
}
public static void Main(string[] args)
{
#if NETCOREAPP3_0
Console.Write("CoFlows CE - NetCoreApp 3.0... ");
#endif
#if NET461
Console.Write("CoFlows CE - Net Framework 461... ");
#endif
Console.Write("Python starting... ");
PythonEngine.Initialize();
Code.InitializeCodeTypes(new Type[] {
typeof(QuantApp.Engine.WorkSpace),
typeof(Jint.Native.Array.ArrayConstructor)
});
var config_env = Environment.GetEnvironmentVariable("coflows_config");
var config_file = Environment.GetEnvironmentVariable("config_file");
if (string.IsNullOrEmpty(config_file))
{
config_file = "quantapp_config.json";
}
JObject config = string.IsNullOrEmpty(config_env) ? (JObject)JToken.ReadFrom(new JsonTextReader(File.OpenText(@"mnt/" + config_file))) : (JObject)JToken.Parse(config_env);
workspace_name = config["Workspace"].ToString();
hostName = config["Server"]["Host"].ToString();
var secretKey = config["Server"]["SecretKey"].ToString();
ssl_cert = config["Server"]["SSL"]["Cert"].ToString();
ssl_password = config["Server"]["SSL"]["Password"].ToString();
var sslFlag = !string.IsNullOrWhiteSpace(ssl_cert);
useJupyter = config["Jupyter"].ToString().ToLower() == "true";
var connectionString = config["Database"].ToString();
var cloudHost = config["Cloud"]["Host"].ToString();
var cloudKey = config["Cloud"]["SecretKey"].ToString();
var cloudSSL = config["Cloud"]["SSL"].ToString();
if (args != null && args.Length > 0 && args[0] == "lab")
{
Connection.Client.Init(hostName, sslFlag);
if (!Connection.Client.Login(secretKey))
{
throw new Exception("CoFlows Not connected!");
}
Connection.Client.Connect();
Console.Write("server connected! ");
QuantApp.Kernel.M.Factory = new MFactory();
var pargs = new string[] { "-m", "ipykernel_launcher.py", "-f", args[1] };
Console.Write("Starting lab... ");
Python.Runtime.Runtime.Py_Main(pargs.Length, pargs);
Console.WriteLine("started lab... ");
}
//Cloud
else if (args != null && args.Length > 1 && args[0] == "cloud" && args[1] == "deploy")
{
Console.WriteLine("Cloud Host: " + cloudHost);
Console.WriteLine("Cloud SSL: " + cloudSSL);
Connection.Client.Init(cloudHost, cloudSSL.ToLower() == "true");
if (!Connection.Client.Login(cloudKey))
{
throw new Exception("CoFlows Not connected!");
}
Connection.Client.Connect();
Console.Write("server connected! ");
QuantApp.Kernel.M.Factory = new MFactory();
Console.Write("Starting cloud deployment... ");
Code.UpdatePackageFile(workspace_name);
var t0 = DateTime.Now;
Console.WriteLine("Started: " + t0);
var res = Connection.Client.PublishPackage(workspace_name);
var t1 = DateTime.Now;
Console.WriteLine("Ended: " + t1 + " taking " + (t1 - t0));
Console.Write("Result: " + res);
}
else if (args != null && args.Length > 1 && args[0] == "cloud" && args[1] == "build")
{
Console.WriteLine("Cloud Host: " + cloudHost);
Console.WriteLine("Cloud SSL: " + cloudSSL);
Connection.Client.Init(cloudHost, cloudSSL.ToLower() == "true");
if (!Connection.Client.Login(cloudKey))
{
throw new Exception("CoFlows Not connected!");
}
Connection.Client.Connect();
Console.Write("server connected! ");
QuantApp.Kernel.M.Factory = new MFactory();
Console.Write("CoFlows Cloud build... ");
Code.UpdatePackageFile(workspace_name);
var t0 = DateTime.Now;
Console.WriteLine("Started: " + t0);
var res = Connection.Client.BuildPackage(workspace_name);
var t1 = DateTime.Now;
Console.WriteLine("Ended: " + t1 + " taking " + (t1 - t0));
Console.Write("Result: " + res);
}
else if (args != null && args.Length > 2 && args[0] == "cloud" && args[1] == "query")
{
Console.WriteLine("Cloud Host: " + cloudHost);
Console.WriteLine("Cloud SSL: " + cloudSSL);
Connection.Client.Init(cloudHost, cloudSSL.ToLower() == "true");
if (!Connection.Client.Login(cloudKey))
{
throw new Exception("CoFlows Not connected!");
}
Connection.Client.Connect();
Console.Write("server connected! ");
QuantApp.Kernel.M.Factory = new MFactory();
Console.WriteLine("CoFlows Cloud query... ");
var queryID = args[2];
var funcName = args.Length > 3 ? args[3] : null;
var parameters = args.Length > 4 ? args.Skip(4).ToArray() : null;
var pkg = Code.ProcessPackageFile(workspace_name);
Console.WriteLine("Workspace: " + pkg.Name);
Console.WriteLine("Query ID: " + queryID);
Console.WriteLine("Function Name: " + funcName);
if (parameters != null)
{
for (int i = 0; i < parameters.Length; i++)
{
Console.WriteLine("Parameter[" + i + "]: " + parameters[i]);
}
}
var(code_name, code) = pkg.Queries.Where(entry => entry.ID == queryID).Select(entry => (entry.Name as string, entry.Content as string)).FirstOrDefault();
var t0 = DateTime.Now;
Console.WriteLine("Started: " + t0);
var result = Connection.Client.Execute(code, code_name, pkg.ID, queryID, funcName, parameters);
var t1 = DateTime.Now;
Console.WriteLine("Ended: " + t1 + " taking " + (t1 - t0));
Console.WriteLine("Result: ");
Console.WriteLine(result);
}
else if (args != null && args.Length > 1 && args[0] == "cloud" && args[1] == "log")
{
Console.WriteLine("Cloud Host: " + cloudHost);
Console.WriteLine("Cloud SSL: " + cloudSSL);
Connection.Client.Init(cloudHost, cloudSSL.ToLower() == "true");
if (!Connection.Client.Login(cloudKey))
{
throw new Exception("CoFlows Not connected!");
}
Connection.Client.Connect();
Console.Write("server connected! ");
QuantApp.Kernel.M.Factory = new MFactory();
Console.Write("CoFlows Cloud log... ");
var res = Connection.Client.RemoteLog(workspace_name);
Console.WriteLine("Result: ");
Console.WriteLine(res);
}
else if (args != null && args.Length > 1 && args[0] == "cloud" && args[1] == "remove")
{
Console.WriteLine("Cloud Host: " + cloudHost);
Console.WriteLine("Cloud SSL: " + cloudSSL);
Connection.Client.Init(cloudHost, cloudSSL.ToLower() == "true");
if (!Connection.Client.Login(cloudKey))
{
throw new Exception("CoFlows Not connected!");
}
Connection.Client.Connect();
Console.Write("server connected! ");
QuantApp.Kernel.M.Factory = new MFactory();
Console.Write("CoFlows Cloud log... ");
var res = Connection.Client.RemoteRemove(workspace_name);
Console.WriteLine("Result: ");
Console.WriteLine(res);
}
else if (args != null && args.Length > 1 && args[0] == "cloud" && args[1] == "restart")
{
Console.WriteLine("Cloud Host: " + cloudHost);
Console.WriteLine("Cloud SSL: " + cloudSSL);
Connection.Client.Init(cloudHost, cloudSSL.ToLower() == "true");
if (!Connection.Client.Login(cloudKey))
{
throw new Exception("CoFlows Not connected!");
}
Connection.Client.Connect();
Console.Write("server connected! ");
QuantApp.Kernel.M.Factory = new MFactory();
Console.Write("CoFlows Cloud log... ");
var res = Connection.Client.RemoteRestart(workspace_name);
Console.WriteLine("Result: ");
Console.WriteLine(res);
}
else if (args != null && args.Length > 0 && args[0] == "server")
{
PythonEngine.BeginAllowThreads();
Databases(connectionString);
Console.WriteLine("QuantApp Server " + DateTime.Now);
Console.WriteLine("DB Connected");
Console.WriteLine("Local deployment");
if (string.IsNullOrEmpty(config_env))
{
var pkg = Code.ProcessPackageFile(workspace_name);
Code.ProcessPackageJSON(pkg);
SetDefaultWorkSpaces(new string[] { pkg.ID });
Console.WriteLine(pkg.Name + " started");
}
else
{
Console.WriteLine("Empty server...");
}
#if NETCOREAPP3_0
if (!sslFlag)
{
Init(new string[] { "--urls", "http://*:80" });
}
else
{
Init(args);
}
#endif
#if NET461
Init(new string[] { "--urls", "http://*:80" });
#endif
Task.Factory.StartNew(() => {
while (true)
{
// Console.WriteLine(DateTime.Now.ToString());
System.Threading.Thread.Sleep(1000);
}
});
Console.CancelKeyPress += new ConsoleCancelEventHandler(OnExit);
_closing.WaitOne();
}
//Local
else if (args != null && args.Length > 1 && args[0] == "local" && args[1] == "build")
{
PythonEngine.BeginAllowThreads();
Databases(connectionString);
Console.WriteLine("DB Connected");
Console.WriteLine("Local build");
var pkg = Code.ProcessPackageFile(Code.UpdatePackageFile(workspace_name));
var res = Code.BuildRegisterPackage(pkg);
if (string.IsNullOrEmpty(res))
{
Console.WriteLine("Success!!!");
}
else
{
Console.WriteLine(res);
}
}
else if (args != null && args.Length > 2 && args[0] == "local" && args[1] == "query")
{
PythonEngine.BeginAllowThreads();
Databases(connectionString);
Console.WriteLine("Local Query " + DateTime.Now);
Console.WriteLine("DB Connected");
Console.WriteLine("CoFlows Local query... ");
var queryID = args[2];
var funcName = args.Length > 3 ? args[3] : null;
var parameters = args.Length > 4 ? args.Skip(4).ToArray() : null;
Console.WriteLine("QueryID: " + queryID);
Console.WriteLine("FuncName: " + funcName);
Console.WriteLine("Parameters: " + parameters);
var pkg = Code.ProcessPackageFile(Code.UpdatePackageFile(workspace_name));
Code.ProcessPackageJSON(pkg);
if (parameters != null)
{
for (int i = 0; i < parameters.Length; i++)
{
Console.WriteLine("Parameter[" + i + "]: " + parameters[i]);
}
}
var(code_name, code) = pkg.Queries.Where(entry => entry.ID == queryID).Select(entry => (entry.Name as string, entry.Content as string)).FirstOrDefault();
var t0 = DateTime.Now;
Console.WriteLine("Started: " + t0);
// var wb = wb_res.FirstOrDefault() as CodeData;
var codes = new List <Tuple <string, string> >();
codes.Add(new Tuple <string, string>(code_name, code));
var result = QuantApp.Engine.Utils.ExecuteCodeFunction(false, codes, funcName, parameters);
//var result = Connection.Client.Execute(code, code_name, pkg.ID, queryID, funcName, parameters);
var t1 = DateTime.Now;
Console.WriteLine("Ended: " + t1 + " taking " + (t1 - t0));
Console.WriteLine("Result: ");
Console.WriteLine(result);
}
//Azure Container Instance
else if (args != null && args.Length > 1 && args[0] == "azure" && args[1] == "deploy")
{
PythonEngine.BeginAllowThreads();
Console.WriteLine();
Console.WriteLine("Azure Container Instance start...");
var t0 = DateTime.Now;
Console.WriteLine("Started: " + t0);
var pkg = Code.ProcessPackageFile(Code.UpdatePackageFile(workspace_name));
var res = Code.BuildRegisterPackage(pkg);
if (string.IsNullOrEmpty(res))
{
Console.WriteLine("Build Success!!!");
}
else
{
Console.WriteLine(res);
}
AzureCredentials credentials = SdkContext.AzureCredentialsFactory.FromFile(config["AzureContainerInstance"]["AuthFile"].ToString());
var azure = Azure
.Configure()
.WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic)
.Authenticate(credentials)
.WithDefaultSubscription();
string rgName = pkg.ID.ToLower() + "-rg";
string aciName = pkg.Name.ToLower();
string containerImageName = "coflows/ce";
Console.WriteLine("Container Name: " + aciName);
Console.WriteLine("Resource Group Name: " + rgName);
try
{
Console.WriteLine("Cleaning Resource Group: " + rgName);
azure.ResourceGroups.BeginDeleteByName(rgName);
IResourceGroup resGroup = azure.ResourceGroups.GetByName(rgName);
while (resGroup != null)
{
resGroup = azure.ResourceGroups.GetByName(rgName);
Console.Write(".");
SdkContext.DelayProvider.Delay(1000);
}
Console.WriteLine();
Console.WriteLine("Cleaned Resource Group: " + rgName);
}
catch (Exception e)
{
Console.WriteLine();
Console.WriteLine("Did not create any resources in Azure. No clean up is necessary");
}
Region region = Region.Create(config["AzureContainerInstance"]["Region"].ToString());
Console.WriteLine("Region: " + region);
if (config["AzureContainerInstance"]["Gpu"] != null && config["AzureContainerInstance"]["Gpu"]["Cores"].ToString() != "" && config["AzureContainerInstance"]["Gpu"]["Cores"].ToString() != "0" && config["AzureContainerInstance"]["Gpu"]["SKU"].ToString() != "")
{
Console.WriteLine("Creating a GPU container...");
Task.Run(() =>
azure.ContainerGroups.Define(aciName)
.WithRegion(region)
.WithNewResourceGroup(rgName)
.WithLinux()
.WithPublicImageRegistryOnly()
// .WithNewAzureFileShareVolume(volumeMountName, shareName)
.WithoutVolume()
.DefineContainerInstance(aciName)
.WithImage(containerImageName)
.WithExternalTcpPort(sslFlag ? 443 : 80)
// .WithVolumeMountSetting(volumeMountName, "/aci/logs/")
.WithCpuCoreCount(Int32.Parse(config["AzureContainerInstance"]["Cores"].ToString()))
.WithMemorySizeInGB(Int32.Parse(config["AzureContainerInstance"]["Mem"].ToString()))
.WithGpuResource(
Int32.Parse(config["AzureContainerInstance"]["Gpu"]["Cores"].ToString()),
config["AzureContainerInstance"]["Gpu"]["SKU"].ToString().ToLower() == "k80" ? GpuSku.K80 : config["AzureContainerInstance"]["Gpu"]["SKU"].ToString().ToLower() == "p100" ? GpuSku.P100 : GpuSku.V100
)
.WithEnvironmentVariables(new Dictionary <string, string>()
{
{ "coflows_config", File.ReadAllText(@"mnt/" + config_file) },
})
.WithStartingCommandLine("dotnet", "QuantApp.Server.quant.lnx.dll", "server")
.Attach()
.WithDnsPrefix(config["AzureContainerInstance"]["Dns"].ToString())
.CreateAsync()
);
}
else
{
Console.WriteLine("Creating a standard container...");
Task.Run(() =>
azure.ContainerGroups.Define(aciName)
.WithRegion(region)
.WithNewResourceGroup(rgName)
.WithLinux()
.WithPublicImageRegistryOnly()
// .WithNewAzureFileShareVolume(volumeMountName, shareName)
.WithoutVolume()
.DefineContainerInstance(aciName)
.WithImage(containerImageName)
.WithExternalTcpPort(sslFlag ? 443 : 80)
// .WithVolumeMountSetting(volumeMountName, "/aci/logs/")
.WithCpuCoreCount(Int32.Parse(config["AzureContainerInstance"]["Cores"].ToString()))
.WithMemorySizeInGB(Int32.Parse(config["AzureContainerInstance"]["Mem"].ToString()))
.WithEnvironmentVariables(new Dictionary <string, string>()
{
{ "coflows_config", File.ReadAllText(@"mnt/" + config_file) },
})
.WithStartingCommandLine("dotnet", "QuantApp.Server.quant.lnx.dll", "server")
.Attach()
.WithDnsPrefix(config["AzureContainerInstance"]["Dns"].ToString())
.CreateAsync()
);
}
// Poll for the container group
IContainerGroup containerGroup = null;
while (containerGroup == null)
{
containerGroup = azure.ContainerGroups.GetByResourceGroup(rgName, aciName);
Console.Write(".");
SdkContext.DelayProvider.Delay(1000);
}
var lastContainerGroupState = containerGroup.Refresh().State;
Console.WriteLine();
Console.WriteLine($"Container group state: {containerGroup.Refresh().State}");
// Poll until the container group is running
while (containerGroup.State != "Running")
{
var containerGroupState = containerGroup.Refresh().State;
if (containerGroupState != lastContainerGroupState)
{
Console.WriteLine();
Console.WriteLine(containerGroupState);
lastContainerGroupState = containerGroupState;
}
Console.Write(".");
System.Threading.Thread.Sleep(1000);
}
Console.WriteLine();
Console.WriteLine("Container instance IP address: " + containerGroup.IPAddress);
Console.WriteLine("Container instance Ports: " + string.Join(",", containerGroup.ExternalTcpPorts));
string serverUrl = config["AzureContainerInstance"]["Dns"].ToString() + "." + config["AzureContainerInstance"]["Region"].ToString().ToLower() + ".azurecontainer.io";
Console.WriteLine("Container instance DNS Prefix: " + serverUrl);
SdkContext.DelayProvider.Delay(10000);
Connection.Client.Init(serverUrl, sslFlag);
if (!Connection.Client.Login(config["Server"]["SecretKey"].ToString()))
{
throw new Exception("CoFlows Not connected!");
}
Connection.Client.Connect();
Console.Write("Container connected! ");
QuantApp.Kernel.M.Factory = new MFactory();
Console.Write("Starting azure deployment... ");
Code.UpdatePackageFile(workspace_name);
var resDeploy = Connection.Client.PublishPackage(workspace_name);
var t1 = DateTime.Now;
Console.WriteLine("Ended: " + t1 + " taking " + (t1 - t0));
Console.Write("Result: " + resDeploy);
}
else if (args != null && args.Length > 1 && args[0] == "azure" && args[1] == "remove")
{
PythonEngine.BeginAllowThreads();
Console.WriteLine("Azure Container Instance remove start");
var pkg = Code.ProcessPackageFile(Code.UpdatePackageFile(workspace_name));
var res = Code.BuildRegisterPackage(pkg);
if (!string.IsNullOrEmpty(res))
{
Console.WriteLine(res);
}
AzureCredentials credentials = SdkContext.AzureCredentialsFactory.FromFile(config["AzureContainerInstance"]["AuthFile"].ToString());
var azure = Azure
.Configure()
.WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic)
.Authenticate(credentials)
.WithDefaultSubscription();
string rgName = pkg.ID.ToLower() + "-rg";
try
{
Console.WriteLine("Deleting Resource Group: " + rgName);
azure.ResourceGroups.BeginDeleteByName(rgName);
IResourceGroup resGroup = azure.ResourceGroups.GetByName(rgName);
while (resGroup != null)
{
resGroup = azure.ResourceGroups.GetByName(rgName);
Console.Write(".");
SdkContext.DelayProvider.Delay(1000);
}
Console.WriteLine();
Console.WriteLine("Deleted Resource Group: " + rgName);
}
catch (Exception)
{
Console.WriteLine("Did not create any resources in Azure. No clean up is necessary");
}
}
else
{
Console.WriteLine("Wrong argument");
}
}
private static DateTime GetRGCreationDateTime(
AzureCredentials credentials,
Microsoft.Azure.Management.ResourceManager.ResourceManagementClient client,
IResourceGroup rg)
{
var url = $"{client.BaseUri.AbsoluteUri}subscriptions/{client.SubscriptionId}/resourceGroups/{rg.Name}?$expand=createdTime&api-version={client.ApiVersion}";
var request = new HttpRequestMessage(HttpMethod.Get, url);
credentials.ProcessHttpRequestAsync(request, CancellationToken.None)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
var response = client.HttpClient.SendAsync(request)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
if (response.StatusCode != HttpStatusCode.OK)
{
throw new Exception($"Resource Group '{rg.Name}' did not return creation date. Skipping.");
}
string responseContent = response.Content.ReadAsStringAsync()
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
JObject body = null;
if (!string.IsNullOrWhiteSpace(responseContent))
{
body = JObject.Parse(responseContent);
}
else
{
throw new Exception($"Resource Group '{rg.Name}' did not provide body content. Skipping.");
}
if (body == null ||
body["createdTime"] == null)
{
throw new Exception($"Resource Group '{rg.Name}' did not return creation date in the body message. Skipping.");
}
var settings = new JsonSerializerSettings
{
Formatting = Formatting.Indented,
DateFormatHandling = DateFormatHandling.IsoDateFormat,
DateTimeZoneHandling = DateTimeZoneHandling.Utc,
NullValueHandling = NullValueHandling.Ignore,
ReferenceLoopHandling = ReferenceLoopHandling.Serialize,
ContractResolver = new ReadOnlyJsonContractResolver(),
Converters = new List <JsonConverter>
{
new Iso8601TimeSpanConverter()
}
};
string str = SafeJsonConvert.SerializeObject(body["createdTime"], settings).Trim(new[] { '\"' });
return(DateTime.Parse(str));
}
public async System.Threading.Tasks.Task GetSecrets()
{
_clientId = GetAppSetting("clientId");
_clientSecret = GetAppSetting("clientSecret");
_tenantId = GetAppSetting("tenantId");
_subId = GetAppSetting("subId");
_credentials = SdkContext.AzureCredentialsFactory.FromServicePrincipal(_clientId,
_clientSecret,
_tenantId,
AzureEnvironment.AzureGlobalCloud);
KeyVaultClient kvClient = new KeyVaultClient(
async(string authority, string resource, string scope) =>
{
AuthenticationContext authContext = new AuthenticationContext(authority);
ClientCredential clientCred = new ClientCredential(_clientId, _clientSecret);
AuthenticationResult result = await authContext.AcquireTokenAsync(resource, clientCred);
if (result == null)
{
throw new InvalidOperationException("Failed to retrieve access token for Key Vault");
}
return(result.AccessToken);
});
_gitToken = (await kvClient.GetSecretAsync("https://appsvcbuild-vault.vault.azure.net/", "gitToken")).Value;
_sendGridApiKey = (await kvClient.GetSecretAsync("https://appsvcbuild-vault.vault.azure.net/", "sendGridApiKey")).Value;
_blimpfuncMaster = (await kvClient.GetSecretAsync("https://appsvcbuild-vault.vault.azure.net/", "blimpfuncMaster")).Value;
_acrPassword = (await kvClient.GetSecretAsync("https://appsvcbuild-vault.vault.azure.net/", "acrPassword")).Value;
_pipelineToken = (await kvClient.GetSecretAsync("https://appsvcbuild-vault.vault.azure.net/", "pipelineToken")).Value;
if (_clientId == "")
{
throw new Exception("missing appsetting clientId ");
}
if (_clientSecret == "")
{
throw new Exception("missing appsetting clientSecret ");
}
if (_tenantId == "")
{
throw new Exception("missing appsetting tenantId ");
}
if (_subId == "")
{
throw new Exception("missing appsetting subId ");
}
if (_gitToken == "")
{
throw new Exception("missing setting gitToken in keyvault");
}
if (_sendGridApiKey == "")
{
throw new Exception("missing setting gitToken in sendGridApiKey");
}
if (_blimpfuncMaster == "")
{
throw new Exception("missing setting gitToken in blimpfuncMaster");
}
if (_acrPassword == "")
{
throw new Exception("missing setting gitToken in blimpfuncMaster");
}
if (_pipelineToken == "")
{
throw new Exception("missing setting gitToken in blimpfuncMaster");
}
return;
}
public IGraphRbacManager Authenticate(AzureCredentials credentials, string tenantId)
{
return(new GraphRbacManager(BuildRestClient(credentials), tenantId));
}
public async Task <bool> DeployAsync()
{
var isDeploymentSuccessful = false;
var mainTimer = Stopwatch.StartNew();
RefreshableConsole.WriteLine("Running...");
await ValidateTokenProviderAsync();
tokenCredentials = new TokenCredentials(new RefreshableAzureServiceTokenProvider("https://management.azure.com/"));
azureCredentials = new AzureCredentials(tokenCredentials, null, null, AzureEnvironment.AzureGlobalCloud);
azureClient = GetAzureClient(azureCredentials);
await ValidateConfigurationAsync();
try
{
RefreshableConsole.WriteLine();
RefreshableConsole.WriteLine($"VM host: {configuration.VmName}.{configuration.RegionName}.cloudapp.azure.com");
RefreshableConsole.WriteLine($"VM username: {configuration.VmUsername}");
RefreshableConsole.WriteLine($"VM password: {configuration.VmPassword}");
RefreshableConsole.WriteLine();
await CreateResourceGroupAsync();
BatchAccount batchAccount = null;
IGenericResource appInsights = null;
ICosmosDBAccount cosmosDb = null;
IStorageAccount storageAccount = null;
IVirtualMachine linuxVm = null;
ConnectionInfo sshConnectionInfo = null;
await Task.WhenAll(new Task[]
{
Task.Run(async() => batchAccount = await CreateBatchAccountAsync(), cts.Token),
Task.Run(async() => appInsights = await CreateAppInsightsResourceAsync(), cts.Token),
Task.Run(async() => cosmosDb = await CreateCosmosDbAsync(), cts.Token),
Task.Run(async() => storageAccount = await CreateStorageAccountAsync(), cts.Token),
Task.Run(async() =>
{
linuxVm = await CreateVirtualMachineAsync();
sshConnectionInfo = new ConnectionInfo(linuxVm.GetPrimaryPublicIPAddress().Fqdn, configuration.VmUsername, new PasswordAuthenticationMethod(configuration.VmUsername, configuration.VmPassword));
await WaitForSshConnectivityAsync(sshConnectionInfo);
await ConfigureVmAsync(sshConnectionInfo);
}, cts.Token)
});
var vmManagedIdentity = linuxVm.SystemAssignedManagedServiceIdentityPrincipalId;
await AssignVmAsBillingReaderToSubscriptionAsync(vmManagedIdentity);
await AssignVmAsContributorToAppInsightsAsync(vmManagedIdentity, appInsights);
await AssignVmAsContributorToCosmosDb(vmManagedIdentity, cosmosDb);
await AssignVmAsContributorToBatchAccountAsync(vmManagedIdentity, batchAccount);
await AssignVmAsContributorToStorageAccountAsync(vmManagedIdentity, storageAccount);
await AssignVmAsDataReaderToStorageAccountAsync(vmManagedIdentity, storageAccount);
await RestartVmAsync(linuxVm);
await WaitForSshConnectivityAsync(sshConnectionInfo);
await WaitForDockerComposeAsync(sshConnectionInfo);
await WaitForCromwellAsync(sshConnectionInfo);
isDeploymentSuccessful = await VerifyInstallationAsync(storageAccount);
if (!isDeploymentSuccessful)
{
await DeleteResourceGroupIfUserConsentsAsync();
}
}
catch (Microsoft.Rest.Azure.CloudException cloudException)
{
var json = cloudException.Response.Content;
RefreshableConsole.WriteLine(json);
Debugger.Break();
WriteGeneralRetryMessageToConsole();
await DeleteResourceGroupIfUserConsentsAsync();
}
catch (Exception exc)
{
RefreshableConsole.WriteLine(exc.ToString());
Debugger.Break();
WriteGeneralRetryMessageToConsole();
await DeleteResourceGroupIfUserConsentsAsync();
}
RefreshableConsole.WriteLine($"Completed in {mainTimer.Elapsed.TotalMinutes:n1} minutes.");
return(isDeploymentSuccessful);
}
/// Constructor
/// </summary>
/// <param name="credentials">Credential to authenticate to a subscription</param>
/// <param name="subscriptionId">Subscription Id</param>
public NetworkInterfaces(AzureCredentials credentials, string subscriptionId) : base(credentials, subscriptionId)
{
}
public ServiceBusQueueScraper(AzureMetadata azureMetadata, AzureCredentials azureCredentials) : base(azureMetadata, azureCredentials)
{
}
public ProviderRegistrationDelegatingHandler(AzureCredentials credentials, HttpMessageHandler innerHandler)
: base(innerHandler)
{
this.credentials = credentials;
}
public async Task <IActionResult> TestPrepareDedicatedHostGroup(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
var parameters = req.GetQueryParameterDictionary();
if (!parameters.ContainsKey(ResourceGroupName) || string.IsNullOrEmpty(parameters[ResourceGroupName]))
{
return(new BadRequestObjectResult("Resource group name was missing in the query parameters."));
}
if (!parameters.ContainsKey(HostGroupName) || string.IsNullOrEmpty(parameters[HostGroupName]))
{
return(new BadRequestObjectResult("Host group name was missing in the query parameters."));
}
if (!parameters.ContainsKey(VmCount) || !Int32.TryParse(parameters[VmCount], out int numVirtualMachines))
{
return(new BadRequestObjectResult("VmCount was missing in the query parameters."));
}
if (!parameters.ContainsKey(VmSku) || string.IsNullOrEmpty(parameters[VmSku]))
{
return(new BadRequestObjectResult("VM SKU was missing in the query parameters."));
}
var authEndpoint = _configuration["AuthEndpoint"];
var azureRmEndpoint = _configuration["AzureRmEndpoint"];
var location = _configuration["Location"];
var virtualMachineSize = parameters[VmSku];
var tenantId = _configuration["TenantId"];
var clientId = _configuration["ClientId"];
var clientSecret = _configuration["FairfaxClientSecret"];
var subscriptionId = _configuration["SubscriptionId"];
var resourceGroupName = parameters[ResourceGroupName];
var hostGroupName = parameters[HostGroupName];
log.LogInformation($"Generating auth token...");
var token = await TokenHelper.GetToken(
authEndpoint,
azureRmEndpoint,
tenantId,
clientId,
clientSecret);
var customTokenProvider = new AzureCredentials(
new TokenCredentials(token),
new TokenCredentials(token),
tenantId,
AzureEnvironment.FromName(_configuration["CloudName"]));
var client = RestClient
.Configure()
.WithEnvironment(AzureEnvironment.FromName(_configuration["CloudName"]))
.WithLogLevel(HttpLoggingDelegatingHandler.Level.Basic)
.WithCredentials(customTokenProvider)
.Build();
var azure = Azure.Authenticate(client, tenantId).WithSubscription(subscriptionId);
var computeManagementClient = new ComputeManagementClient(customTokenProvider)
{
SubscriptionId = subscriptionId,
BaseUri = new Uri(_configuration["ResourceManagerUri"]),
LongRunningOperationRetryTimeout = 5
};
log.LogInformation($"Creating resource group ({resourceGroupName}), if needed");
var resourceGroup = azure.ResourceGroups.Define(resourceGroupName)
.WithRegion(location)
.Create();
log.LogInformation($"Creating host group ({hostGroupName}), if needed");
var newDedicatedHostGroup = new DedicatedHostGroup()
{
Location = location,
PlatformFaultDomainCount = 1
};
await computeManagementClient.DedicatedHostGroups.CreateOrUpdateAsync(
resourceGroupName,
hostGroupName,
newDedicatedHostGroup);
#if DEBUG
var prepareDHGroup =
$"http://localhost:7071/api/PrepareDedicatedHostGroup" +
$"?token={token}" +
$"&cloudName={_configuration["CloudName"]}" +
$"&tenantId={tenantId}" +
$"&subscriptionId={subscriptionId}" +
$"&resourceGroup={resourceGroupName}" +
$"&vmSku={virtualMachineSize}" +
$"&dedicatedHostGroupName={hostGroupName}" +
$"&vmCount={numVirtualMachines}" +
$"&platformFaultDomain=0";
#else
var prepareDHGroup =
_configuration["PrepareDHGroupUri"] +
$"&token={token}" +
$"&cloudName={_configuration["CloudName"]}" +
$"&tenantId={tenantId}" +
$"&subscriptionId={subscriptionId}" +
$"&resourceGroup={resourceGroupName}" +
$"&vmSku={virtualMachineSize}" +
$"&dedicatedHostGroupName={hostGroupName}" +
$"&vmCount={numVirtualMachines}" +
$"&platformFaultDomain=0";
#endif
var response = await _httpClient.GetAsync(prepareDHGroup);
if (response.StatusCode != HttpStatusCode.OK)
{
return(new ObjectResult(new { error = $"Exception thrown by {await response.Content.ReadAsStringAsync()}" })
{
StatusCode = (int)response.StatusCode
});
}
var dhCreated = await response.Content.ReadAsAsync <List <DedicatedHost> >();
return(new OkObjectResult($"Prepared Dedicated Host Group completed successfully {string.Join(",", dhCreated.Select(c => c.Name))} VM."));
}
