public async Task <bool> WaitForComputeResourceDeploymentAsync(string resourceGroupName, string resourceName, string resourceType) { bool resourceExists = false; while (!resourceExists) { resourceExists = _resourceManager.GenericResources.CheckExistence(resourceGroupName, "Microsoft.Compute", String.Empty, resourceType, resourceName, "2016-04-30-preview"); await Task.Delay(TimeSpan.FromSeconds(5)); } IGenericResource computeResource = GetComputeResource(resourceGroupName, resourceName, resourceType); ProvisioningState provisioningState = ProvisioningState.Parse((string)JObject.FromObject(computeResource.Properties)["provisioningState"]); while (!provisioningState.DeploymentEnded()) { await SdkContext.DelayProvider.DelayAsync(DeploymentDelayMilliseconds, CancellationToken.None); computeResource = GetComputeResource(resourceGroupName, resourceName, resourceType); provisioningState = ProvisioningState.Parse( (string)JObject.FromObject(computeResource.Properties)["provisioningState"]); } return(provisioningState.DeploymentSucceeded()); }
protected async Task <IPublicIPAddress> DeployPublicIp(ICollection <IGenericResource> unexpectedResources, IAzure azure, string name, string domainName, CancellationToken cancellationToken) { IGenericResource existingIp = unexpectedResources.FirstOrDefault(r => string.Equals(r.ResourceProviderNamespace, "Microsoft.Network", StringComparison.OrdinalIgnoreCase) && string.Equals(r.ResourceType, "publicIPAddresses", StringComparison.OrdinalIgnoreCase) && string.Equals(r.Name, name, StringComparison.OrdinalIgnoreCase)); if (existingIp == null) { Logger.LogInformation("Creating new IP address {ipName}", name); return(await azure.PublicIPAddresses.Define(name) .WithRegion(Settings.Location) .WithExistingResourceGroup(Settings.ResourceGroup) .WithTags(DefaultTags) .WithSku(PublicIPSkuType.Standard) .WithStaticIP() .WithLeafDomainLabel(domainName) .CreateAsync(cancellationToken)); } unexpectedResources.Remove(existingIp); Logger.LogInformation("Updating existing IP address {ipName}", name); return(await(await azure.PublicIPAddresses.GetByResourceGroupAsync(Settings.ResourceGroup, name, cancellationToken)) .Update() .WithTags(DefaultTags) .WithStaticIP() .WithLeafDomainLabel(domainName) .ApplyAsync(cancellationToken)); }
public void CanUpdateVirtualNetworkDeployment() { using (var context = FluentMockContext.Start(this.GetType().FullName)) { IResourceManager resourceManager = TestHelper.CreateResourceManager(); try { var deployment = resourceManager.Deployments .Define(deploymentName3) .WithNewResourceGroup(rgName, Region.USEast) .WithTemplateLink(templateUri, contentVersion) .WithParametersLink(parametersUri, contentVersion) .WithMode(DeploymentMode.Complete) .BeginCreate(); Assert.Equal(deploymentName3, deployment.Name); deployment.Cancel(); deployment = resourceManager.Deployments.GetByResourceGroup(rgName, deploymentName3); Assert.Equal("Canceled", deployment.ProvisioningState); deployment.Update() .WithTemplate(updateTemplate) .WithParameters(updateParameters) .WithMode(DeploymentMode.Incremental) .Apply(); deployment = resourceManager.Deployments.GetByResourceGroup(rgName, deploymentName3); Assert.True(deployment.Mode == DeploymentMode.Incremental); Assert.Equal("Succeeded", deployment.ProvisioningState); IGenericResource genericVnet = resourceManager.GenericResources.Get( rgName, "Microsoft.Network", "", "virtualnetworks", "VNet2", "2015-06-15"); Assert.NotNull(genericVnet); resourceManager.GenericResources.Delete( rgName, "Microsoft.Network", "", "virtualnetworks", "VNet2", "2015-06-15"); } finally { try { TestHelper.CreateResourceManager().ResourceGroups.DeleteByName(rgName); } catch { } } } }
private async Task <INetwork> DeployVirtualNetwork(ICollection <IGenericResource> unexpectedResources, IAzure azure, INetworkSecurityGroup nsg, CancellationToken cancellationToken) { string vnetName = Settings.Name + "-vnet"; IGenericResource existingNetworkResource = unexpectedResources.FirstOrDefault(r => string.Equals(r.ResourceProviderNamespace, "Microsoft.Network", StringComparison.OrdinalIgnoreCase) && string.Equals(r.ResourceType, "virtualNetworks", StringComparison.OrdinalIgnoreCase) && string.Equals(r.Name, vnetName, StringComparison.OrdinalIgnoreCase)); var pairs = Enumerable.Range(1, 5).SelectMany(i => Enumerable.Range(1, 5).Select(j => (i, j))); var nodeSubnets = pairs.Select(p => ($"Cluster-{p.i}-Node-{p.j}", $"10.{p.i}.{p.j}.0/24")); IEnumerable <(string, string)> neededSubnets = new[]
protected override bool GetResourceFilter(IGenericResource res) { return(res.ResourceType == "Microsoft.Storage/storageAccounts"); }
public static Mearcury.Core.Resource Convert(this IGenericResource resource) { return(new Mearcury.Core.Resource(resource.Id, resource.Name, resource.ResourceGroupName, resource.Type, resource.Tags)); }
async static Task <int> ProcessMetricAsync(IAzure subApi, ISubscription subscription, IResourceGroup group, IGenericResource res, IMetricDefinition metric) { int m = 0; DateTime? last = null; MetricLastDate record = null; lock (_ctx) { record = _ctx.MetricLastDates.Where(d => d.Key == metric.Id).SingleOrDefault(); } last = record?.LastDate; try { await _sem.WaitAsync(); var from = last?.AddMinutes(1.0) ?? DateTime.UtcNow.RoundMinutes().AddHours(-1.0); var to = DateTime.UtcNow.RoundMinutes().AddMinutes(10.0); var maxDataToGet = TimeSpan.FromHours(2.0); if (to - from > maxDataToGet) { to = from + maxDataToGet; } if (metric.Inner.IsDimensionRequired == true) { var dims = metric.Inner.Dimensions; // can ignore.. return(0); } var data = await metric.DefineQuery() .StartingFrom(from) .EndsBefore(to) // as much as possible .WithAggregation("Average,Minimum,Maximum,Count,Total") .WithResultType(ResultType.Data) .WithInterval(TimeSpan.FromMinutes(1)) .ExecuteAsync(); if (data.Metrics.Count != 1 || data.Metrics[0].Timeseries.Count != 1) { return(0); } /* * Console.WriteLine($"query from {from} to {to}: {data.Metrics[0].Timeseries[0].Data.Count()} results"); * Console.WriteLine($" min: {data.Metrics[0].Timeseries[0].Data.Min(d => d.Minimum)}"); * Console.WriteLine($" max: {data.Metrics[0].Timeseries[0].Data.Max(d => d.Maximum)}"); * Console.WriteLine($" avg: {data.Metrics[0].Timeseries[0].Data.Average(d => d.Average)}"); */ var dubiousCutoffUtc = DateTime.UtcNow.AddMinutes(-30.0); var pts = data.Metrics[0].Timeseries[0].Data // from future to past .OrderByDescending(d => d.TimeStamp) // skip anything with no data yet .SkipWhile(d => d.TimeStamp > dubiousCutoffUtc && (d.Average ?? 0) == 0 && (d.Maximum ?? 0) == 0 && (d.Minimum ?? 0) == 0 && (d.Total ?? 0) == 0 && ( (d.Count ?? 0) == 0 || d.TimeStamp > dubiousCutoffUtc ) ) // and the first one with data (probably the current slice) .Skip(1); /* * if (pts.Any()) * Console.WriteLine($"actual data points: {pts.Count()}, from {pts.Min(p => p.TimeStamp)} to {pts.Max(p => p.TimeStamp)}"); * else * Console.WriteLine($"all data points dismissed"); */ if (pts.Any()) { lock (_logger) { foreach (var pt in pts.Reverse()) { var entry = new Dictionary <string, object>(13) { { "@timestamp", pt.TimeStamp.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFFZ") }, { "lib", "Saffron" }, { "type", "cloud" }, { "cloud", "Azure" }, { "sub", subscription.DisplayName }, { "group", group.Name }, { "resource", new Dictionary <string, object>(3) { { "name", res.Name }, { "type", res.Type }, { "tags", res.Tags } } }, { "metric", new Dictionary <string, object>(2) { { "name", metric.Name.Value }, { "unit", metric.Unit } } }, }; if (pt.Average != null) { entry["avg"] = pt.Average; } if (pt.Count != null) { entry["num"] = pt.Count; } if (pt.Maximum != null) { entry["max"] = pt.Maximum; } if (pt.Minimum != null) { entry["min"] = pt.Minimum; } if (pt.Total != null) { entry["sum"] = pt.Total; } _logger.LogAsJson(entry); m++; } } lock (_ctx) { if (record == null) { record = new MetricLastDate { Key = metric.Id }; _ctx.MetricLastDates.Add(record); } record.LastDate = pts.First().TimeStamp; _ctx.SaveChanges(); numStats++; long ageMs = (long)(DateTime.UtcNow - record.LastDate).TotalMilliseconds; totalAgeMs += ageMs; if (ageMs > oldestAgeMs) { oldestAgeMs = ageMs; } } } } catch (ErrorResponseException ex) { // ignored //Console.WriteLine($"Error reading {metric.Name.LocalizedValue} from {res.Name} ({res.Type}): {ex.Message} - {ex.Body.Code} {ex.Body.Message}"); } catch (HttpRequestException ex) { Console.WriteLine($"HTTP error reading {metric.Name.LocalizedValue} from {res.Name} ({res.Type}): {ex.Message}"); } catch (Exception ex) { Console.WriteLine($"Error reading {metric.Name.LocalizedValue} from {res.Name} ({res.Type}): {ex.Message}"); } finally { _sem.Release(); } return(m); }
async static Task <int> ProcessResourceAsync(IAzure subApi, ISubscription sub, IResourceGroup group, IGenericResource res) { try { var metrics = await subApi.MetricDefinitions.ListByResourceAsync(res.Id); var all = await Task.WhenAll(metrics.Select(m => ProcessMetricAsync(subApi, sub, group, res, m))); return(all.Sum()); } catch (Exception ex) { // todo... Console.WriteLine("Exception getting metric defs: " + ex.Message); return(0); } }
public async Task CanCreateUpdateMoveResource() { using (var context = FluentMockContext.Start(this.GetType().FullName)) { var resourceName = TestUtilities.GenerateName("rgweb"); var rgName = TestUtilities.GenerateName("csmrg"); var newRgName = TestUtilities.GenerateName("csmrg"); IResourceManager resourceManager = TestHelper.CreateResourceManager(); IGenericResources genericResources = resourceManager.GenericResources; IGenericResource resource = genericResources.Define(resourceName) .WithRegion(Region.USEast) .WithNewResourceGroup(rgName) .WithResourceType("sites") .WithProviderNamespace("Microsoft.Web") .WithoutPlan() .WithApiVersion("2015-08-01") .WithParentResource("") .WithProperties(JsonConvert.DeserializeObject("{\"SiteMode\":\"Limited\",\"ComputeMode\":\"Shared\"}")) .Create(); // List var found = (from r in await genericResources.ListByResourceGroupAsync(rgName) where string.Equals(r.Name, resourceName, StringComparison.OrdinalIgnoreCase) select r).FirstOrDefault(); Assert.NotNull(found); // Get resource = genericResources.Get(rgName, resource.ResourceProviderNamespace, resource.ParentResourceId, resource.ResourceType, resource.Name, resource.ApiVersion); // Move IResourceGroup newGroup = resourceManager .ResourceGroups .Define(newRgName) .WithRegion(Region.USEast) .Create(); genericResources.MoveResources(rgName, newGroup, new List <string> { resource.Id }); // Check existence [TODO: Server returned "MethodNotAllowed" for CheckExistence call] /*bool exists = genericResources.CheckExistence(newRgName, * resource.ResourceProviderNamespace, * resource.ParentResourceId, * resource.ResourceType, * resource.Name, * resource.ApiVersion); * * Assert.True(exists); */ // Get and update resource = genericResources.Get(newRgName, resource.ResourceProviderNamespace, resource.ParentResourceId, resource.ResourceType, resource.Name, resource.ApiVersion); resource.Update() .WithApiVersion("2015-08-01") .WithProperties(JsonConvert.DeserializeObject("{\"SiteMode\":\"Limited\",\"ComputeMode\":\"Dynamic\"}")) .Apply(); Assert.Equal(newRgName, resource.ResourceGroupName); resourceManager.ResourceGroups.BeginDeleteByName(newRgName); resourceManager.ResourceGroups.BeginDeleteByName(rgName); } }
public IWithPolicyDefinition ForResource(IGenericResource genericResource) { Inner.Scope = genericResource.Id; return(this); }
public async Task <bool> DeployAsync() { var isDeploymentSuccessful = false; var mainTimer = Stopwatch.StartNew(); RefreshableConsole.WriteLine("Running..."); await ValidateTokenProviderAsync(); tokenCredentials = new TokenCredentials(new RefreshableAzureServiceTokenProvider("https://management.azure.com/")); azureCredentials = new AzureCredentials(tokenCredentials, null, null, AzureEnvironment.AzureGlobalCloud); azureClient = GetAzureClient(azureCredentials); await ValidateConfigurationAsync(); try { RefreshableConsole.WriteLine(); RefreshableConsole.WriteLine($"VM host: {configuration.VmName}.{configuration.RegionName}.cloudapp.azure.com"); RefreshableConsole.WriteLine($"VM username: {configuration.VmUsername}"); RefreshableConsole.WriteLine($"VM password: {configuration.VmPassword}"); RefreshableConsole.WriteLine(); await CreateResourceGroupAsync(); BatchAccount batchAccount = null; IGenericResource appInsights = null; ICosmosDBAccount cosmosDb = null; IStorageAccount storageAccount = null; IVirtualMachine linuxVm = null; ConnectionInfo sshConnectionInfo = null; await Task.WhenAll(new Task[] { Task.Run(async() => batchAccount = await CreateBatchAccountAsync(), cts.Token), Task.Run(async() => appInsights = await CreateAppInsightsResourceAsync(), cts.Token), Task.Run(async() => cosmosDb = await CreateCosmosDbAsync(), cts.Token), Task.Run(async() => storageAccount = await CreateStorageAccountAsync(), cts.Token), Task.Run(async() => { linuxVm = await CreateVirtualMachineAsync(); sshConnectionInfo = new ConnectionInfo(linuxVm.GetPrimaryPublicIPAddress().Fqdn, configuration.VmUsername, new PasswordAuthenticationMethod(configuration.VmUsername, configuration.VmPassword)); await WaitForSshConnectivityAsync(sshConnectionInfo); await ConfigureVmAsync(sshConnectionInfo); }, cts.Token) }); var vmManagedIdentity = linuxVm.SystemAssignedManagedServiceIdentityPrincipalId; await AssignVmAsBillingReaderToSubscriptionAsync(vmManagedIdentity); await AssignVmAsContributorToAppInsightsAsync(vmManagedIdentity, appInsights); await AssignVmAsContributorToCosmosDb(vmManagedIdentity, cosmosDb); await AssignVmAsContributorToBatchAccountAsync(vmManagedIdentity, batchAccount); await AssignVmAsContributorToStorageAccountAsync(vmManagedIdentity, storageAccount); await AssignVmAsDataReaderToStorageAccountAsync(vmManagedIdentity, storageAccount); await RestartVmAsync(linuxVm); await WaitForSshConnectivityAsync(sshConnectionInfo); await WaitForDockerComposeAsync(sshConnectionInfo); await WaitForCromwellAsync(sshConnectionInfo); isDeploymentSuccessful = await VerifyInstallationAsync(storageAccount); if (!isDeploymentSuccessful) { await DeleteResourceGroupIfUserConsentsAsync(); } } catch (Microsoft.Rest.Azure.CloudException cloudException) { var json = cloudException.Response.Content; RefreshableConsole.WriteLine(json); Debugger.Break(); WriteGeneralRetryMessageToConsole(); await DeleteResourceGroupIfUserConsentsAsync(); } catch (Exception exc) { RefreshableConsole.WriteLine(exc.ToString()); Debugger.Break(); WriteGeneralRetryMessageToConsole(); await DeleteResourceGroupIfUserConsentsAsync(); } RefreshableConsole.WriteLine($"Completed in {mainTimer.Elapsed.TotalMinutes:n1} minutes."); return(isDeploymentSuccessful); }
private async Task <INetworkSecurityGroup> DeployNetworkSecurityGroup(List <IGenericResource> allResources, IPublicIPAddress gatewayIp, IAzure azure, CancellationToken cancellationToken) { string nsgName = Settings.Name + "-nsg"; IGenericResource nsg = allResources.FirstOrDefault(r => r.ResourceProviderNamespace == "Microsoft.Network" && r.ResourceType == "networkSecurityGroups" && r.Name == nsgName); var neededRules = Settings.NeededSecurityGroupRules ?? throw new InvalidOperationException("Configuration not set correctly."); if (nsg == null) { Logger.LogInformation("Creating new network security group {nsgName}.", nsgName); var nsgDef = azure.NetworkSecurityGroups.Define(nsgName) .WithRegion(Settings.Location) .WithExistingResourceGroup(Settings.ResourceGroup) .WithTags(DefaultTags); var index = 1; foreach ((string key, string range) in neededRules) { nsgDef = nsgDef.DefineRule(key) .AllowInbound() .FromAnyAddress() .FromAnyPort() .ToAnyAddress() .ToPortRanges(range) .WithProtocol(SecurityRuleProtocol.Tcp) .WithPriority(1000 + index) .Attach(); index++; } return(await nsgDef.CreateAsync(cancellationToken)); } allResources.Remove(nsg); Logger.LogInformation("Updating existing network security group {nsgName}.", nsg.Name); INetworkSecurityGroup existingGroup = await azure.NetworkSecurityGroups.GetByIdAsync(nsg.Id, cancellationToken); IEnumerable <string> existingRules = existingGroup.SecurityRules.Keys .Where(ruleName => !ruleName.StartsWith("NRMS")); // Ignore rules created by the microsoft management stuff var updatedNsg = existingGroup.Update(); foreach (string rule in existingRules) { updatedNsg = updatedNsg.WithoutRule(rule); } { var index = 1; foreach ((string key, string range) in neededRules) { updatedNsg = updatedNsg.DefineRule(key) .AllowInbound() .FromAnyAddress() .FromAnyPort() .ToAnyAddress() .ToPortRanges(range) .WithProtocol(SecurityRuleProtocol.Tcp) .WithPriority(1000 + index) .Attach(); index++; } } existingGroup = await updatedNsg.ApplyAsync(cancellationToken); return(existingGroup); }
protected virtual bool GetResourceFilter(IGenericResource res) { return(true); }