public static ClusterCreateParametersExtended GetExtendedClusterCreateParameters(string clusterName, ClusterCreateParameters createParameters) { //Deep copy so the createParameters object isn't touched. ClusterCreateParameters clusterCreateParameters = new ClusterCreateParameters(createParameters); //Convert to extended spec. ClusterCreateParametersExtended extendedParams = new ClusterCreateParametersExtended { Location = clusterCreateParameters.Location, Tags = clusterCreateParameters.Tags, Properties = new ClusterCreateProperties { ClusterDefinition = new ClusterDefinition { Kind = clusterCreateParameters.ClusterType, ComponentVersion = clusterCreateParameters.ComponentVersion, Configurations = GetConfigurations(clusterName, clusterCreateParameters) }, Tier = clusterCreateParameters.ClusterTier, ClusterVersion = clusterCreateParameters.Version, ComputeProfile = new ComputeProfile { Roles = GetRoleCollection(clusterCreateParameters) }, OsType = OSType.Linux, SecurityProfile = clusterCreateParameters.SecurityProfile, StorageProfile = GetStorageProfile(clusterCreateParameters), DiskEncryptionProperties = clusterCreateParameters.DiskEncryptionProperties }, Identity = clusterCreateParameters.ClusterIdentity }; return(extendedParams); }
public static void ValidateSpecConversion(ClusterCreateParameters createParams) { //Convert spec. ClusterCreateParametersExtended extendedParams = CreateParametersConverter.GetExtendedClusterCreateParameters("testCluster", createParams); //Validate properties. Assert.Equal(createParams.Tags, extendedParams.Tags); Assert.Equal(createParams.Location, extendedParams.Location); Assert.Equal(createParams.ClusterType, extendedParams.Properties.ClusterDefinition.Kind); Assert.Equal(createParams.ComponentVersion, extendedParams.Properties.ClusterDefinition.ComponentVersion); Assert.Equal(createParams.ClusterTier, extendedParams.Properties.Tier); Assert.Equal(createParams.Version, extendedParams.Properties.ClusterVersion); Assert.Equal(OSType.Linux, extendedParams.Properties.OsType); Assert.Equal(createParams.SecurityProfile, extendedParams.Properties.SecurityProfile); ValidateStorageProfile(createParams.DefaultStorageInfo, extendedParams.Properties.StorageProfile); //Validate configurations. Dictionary <string, Dictionary <string, string> > configurations = extendedParams.Properties.ClusterDefinition.Configurations as Dictionary <string, Dictionary <string, string> >; Assert.NotNull(configurations); ValidateStorageConfigurations(configurations, createParams.DefaultStorageInfo, createParams.AdditionalStorageAccounts); ValidateClusterCredentialConfigurations(configurations); ValidateAdlConfigurations(configurations, createParams.Principal); ValidateHiveMetastoreConfigurations(configurations, createParams.HiveMetastore); ValidateOozieMetastoreConfigurations(configurations, createParams.OozieMetastore); //Validate roles. ValidateRoles(extendedParams.Properties.ComputeProfile.Roles, createParams); }
public static ClusterCreateParametersExtended AddConfigurations(ClusterCreateParametersExtended cluster, string configurationKey, Dictionary <string, string> configs) { string configurations = cluster.Properties.ClusterDefinition.Configurations; var config = JsonConvert.DeserializeObject <Dictionary <string, Dictionary <String, string> > >(configurations); config.Add(configurationKey, configs); var serializedConfig = JsonConvert.SerializeObject(config); cluster.Properties.ClusterDefinition.Configurations = serializedConfig; return(cluster); }
private void AssertContainerMatchesNameProvided(ClusterCreateParametersExtended createParamsExtended, string name) { Dictionary <string, string> coresiteConfig; if (JsonConvert.DeserializeObject <Dictionary <string, Dictionary <string, string> > >(createParamsExtended.Properties.ClusterDefinition.Configurations).TryGetValue("core-site", out coresiteConfig)) { string value; coresiteConfig.TryGetValue("fs.defaultFS", out value); Assert.True(value.StartsWith("wasb://" + name + "@"), "Container does not match the name provided"); } else { Assert.True(false, "Deserialization error"); } }
/// <summary> /// Validate cluster /// </summary> /// <param name="expectedClustername"></param> /// <param name="expectedParameters"></param> /// <param name="actualCluster"></param> public static void ValidateCluster(string expectedClustername, ClusterCreateParametersExtended expectedParameters, Cluster actualCluster) { Assert.Equal(expectedClustername, actualCluster.Name); Assert.Equal(expectedParameters.Properties.Tier, actualCluster.Properties.Tier); Assert.NotNull(actualCluster.Etag); Assert.EndsWith(expectedClustername, actualCluster.Id); Assert.Equal("Running", actualCluster.Properties.ClusterState); Assert.Equal("Microsoft.HDInsight/clusters", actualCluster.Type); Assert.Equal(expectedParameters.Location, actualCluster.Location); Assert.Equal(expectedParameters.Tags, actualCluster.Tags); Assert.Equal(1, actualCluster.Properties.ConnectivityEndpoints.Count(c => c.Name.Equals("HTTPS", StringComparison.OrdinalIgnoreCase))); Assert.Equal(1, actualCluster.Properties.ConnectivityEndpoints.Count(c => c.Name.Equals("SSH", StringComparison.OrdinalIgnoreCase))); Assert.Equal(expectedParameters.Properties.OsType, actualCluster.Properties.OsType); Assert.Null(actualCluster.Properties.Errors); Assert.Equal(HDInsightClusterProvisioningState.Succeeded, actualCluster.Properties.ProvisioningState); Assert.Equal(expectedParameters.Properties.ClusterDefinition.Kind, actualCluster.Properties.ClusterDefinition.Kind); Assert.Equal(expectedParameters.Properties.ClusterVersion, actualCluster.Properties.ClusterVersion.Substring(0, 3)); Assert.Null(actualCluster.Properties.ClusterDefinition.Configurations); }
/// <summary> /// Create cluster create parameters for ADLS Gen2 relevant tests /// </summary> /// <param name="commonData"></param> /// <param name="storageAccountName"></param> /// <param name="storageResourceId"></param> /// <param name="msiResourceId"></param> /// <param name="createParams"></param> /// <returns></returns> public static ClusterCreateParametersExtended PrepareClusterCreateParamsForADLSv2( this CommonTestFixture commonData, string storageAccountName, string storageResourceId, string msiResourceId, ClusterCreateParametersExtended createParams = null) { var createParamsForADLSv2 = createParams ?? commonData.PrepareClusterCreateParams(); bool isDefault = !createParamsForADLSv2.Properties.StorageProfile.Storageaccounts.Any(); createParamsForADLSv2.Properties.StorageProfile.Storageaccounts.Add( new HDInsightStorageAccount { Name = storageAccountName + commonData.DfsEndpointSuffix, IsDefault = isDefault, FileSystem = commonData.ContainerName.ToLowerInvariant(), ResourceId = storageResourceId, MsiResourceId = msiResourceId } ); var identity = new ClusterIdentity { Type = ResourceIdentityType.UserAssigned, UserAssignedIdentities = new Dictionary <string, ClusterIdentityUserAssignedIdentitiesValue> { { msiResourceId, new ClusterIdentityUserAssignedIdentitiesValue() } } }; if (createParamsForADLSv2.Identity == null) { createParamsForADLSv2.Identity = identity; } else { // At this point, only user-assigned managed identity is supported by HDInsight. // So identity type is not checked. createParamsForADLSv2.Identity.UserAssignedIdentities.Union(identity.UserAssignedIdentities); } return(createParamsForADLSv2); }
private void ResetClusterIdentity(ClusterCreateParametersExtended createParams, string aadAuthority, string dataLakeAudience) { var configuation = (Dictionary <string, Dictionary <string, string> >)createParams.Properties.ClusterDefinition.Configurations; Dictionary <string, string> clusterIdentity; if (!configuation.TryGetValue("clusterIdentity", out clusterIdentity)) { return; } clusterIdentity["clusterIdentity.resourceUri"] = dataLakeAudience; string aadTenantIdWithUrl; clusterIdentity.TryGetValue("clusterIdentity.aadTenantId", out aadTenantIdWithUrl); const string defaultPubliCloudAadAuthority = "https://login.windows.net/"; string newAadTenantIdWithUrl = aadTenantIdWithUrl?.Replace(defaultPubliCloudAadAuthority, aadAuthority); clusterIdentity["clusterIdentity.aadTenantId"] = newAadTenantIdWithUrl; }
public void CanConvertHadoopClusterWithCustomMetastoresToMarchSpecWithConfigurations() { ClusterCreateParameters createParams = GetClusterCreateParamsWithMinRequiredValues(); createParams.HiveMetastore = new Metastore("server.database.windows.net", "hiveDb", "username", "password"); createParams.OozieMetastore = new Metastore("server.database.windows.net", "oozieDb", "username", "password"); Dictionary <string, string> testConfig = new Dictionary <string, string> { { "key", "value" } }; createParams.Configurations.Add(ConfigurationKey.HiveSite, testConfig); createParams.Configurations.Add(ConfigurationKey.HiveEnv, testConfig); createParams.Configurations.Add(ConfigurationKey.OozieEnv, testConfig); createParams.Configurations.Add(ConfigurationKey.OozieSite, testConfig); ClusterCreateParametersExtended extendedParams = CreateParametersConverter.GetExtendedClusterCreateParameters("testCluster", createParams); Dictionary <string, Dictionary <string, string> > configurations = extendedParams.Properties.ClusterDefinition.Configurations as Dictionary <string, Dictionary <string, string> >; Assert.True(configurations.ContainsKey(ConfigurationKey.HiveSite)); Dictionary <string, string> hiveSiteConfig = configurations[ConfigurationKey.HiveSite]; Assert.Equal(5, hiveSiteConfig.Count); }
public override void ExecuteCmdlet() { foreach (var component in ComponentVersion.Where(component => !clusterComponentVersion.ContainsKey(component.Key))) { clusterComponentVersion.Add(component.Key, component.Value); } // Construct Configurations foreach (var config in Configurations.Where(config => !clusterConfigurations.ContainsKey(config.Key))) { clusterConfigurations.Add(config.Key, config.Value); } // Add cluster username/password to gateway config. ClusterCreateHelper.AddClusterCredentialToGatewayConfig(HttpCredential, clusterConfigurations); // Construct OS Profile OsProfile osProfile = ClusterCreateHelper.CreateOsProfile(SshCredential, SshPublicKey); // Construct Virtual Network Profile VirtualNetworkProfile vnetProfile = ClusterCreateHelper.CreateVirtualNetworkProfile(VirtualNetworkId, SubnetName); // Handle storage account StorageProfile storageProfile = new StorageProfile() { Storageaccounts = new List <StorageAccount> { } }; if (StorageAccountType == null || StorageAccountType == StorageType.AzureStorage) { var azureStorageAccount = ClusterCreateHelper.CreateAzureStorageAccount(ClusterName, StorageAccountResourceId, StorageAccountKey, StorageContainer, this.DefaultContext.Environment.StorageEndpointSuffix); storageProfile.Storageaccounts.Add(azureStorageAccount); } else if (StorageAccountType == StorageType.AzureDataLakeStore) { ClusterCreateHelper.AddAzureDataLakeStorageGen1ToCoreConfig(StorageAccountResourceId, StorageRootPath, this.DefaultContext.Environment.AzureDataLakeStoreFileSystemEndpointSuffix, clusterConfigurations); } else if (StorageAccountType == StorageType.AzureDataLakeStorageGen2) { var adlsgen2Account = ClusterCreateHelper.CreateAdlsGen2StorageAccount(ClusterName, StorageAccountResourceId, StorageAccountKey, StorageFileSystem, StorageAccountManagedIdentity, this.DefaultContext.Environment.StorageEndpointSuffix); storageProfile.Storageaccounts.Add(adlsgen2Account); } // Handle additional storage accounts foreach ( var storageAccount in AdditionalStorageAccounts.Where( storageAccount => !clusterAdditionalStorageAccounts.ContainsKey(storageAccount.Key))) { clusterAdditionalStorageAccounts.Add(storageAccount.Key, storageAccount.Value); } ClusterCreateHelper.AddAdditionalStorageAccountsToCoreConfig(clusterAdditionalStorageAccounts, clusterConfigurations); // Handle script action foreach (var action in ScriptActions.Where(action => clusterScriptActions.ContainsKey(action.Key))) { clusterScriptActions.Add(action.Key, action.Value.Select(a => a.GetScriptActionFromPSModel()).ToList()); } // Handle metastore if (OozieMetastore != null) { ClusterCreateHelper.AddOozieMetastoreToConfigurations(OozieMetastore, clusterConfigurations); } if (HiveMetastore != null) { ClusterCreateHelper.AddHiveMetastoreToConfigurations(HiveMetastore, clusterConfigurations); } // Handle ADLSGen1 identity if (!string.IsNullOrEmpty(CertificatePassword)) { if (!string.IsNullOrEmpty(CertificateFilePath)) { CertificateFileContents = File.ReadAllBytes(CertificateFilePath); } ClusterCreateHelper.AddDataLakeStorageGen1IdentityToIdentityConfig( GetApplicationId(ApplicationId), GetTenantId(AadTenantId), CertificateFileContents, CertificatePassword, clusterConfigurations, this.DefaultContext.Environment.ActiveDirectoryAuthority, this.DefaultContext.Environment.DataLakeEndpointResourceId); } // Handle Kafka Rest Proxy KafkaRestProperties kafkaRestProperties = null; if (KafkaClientGroupId != null && KafkaClientGroupName != null) { kafkaRestProperties = new KafkaRestProperties() { ClientGroupInfo = new ClientGroupInfo(KafkaClientGroupName, KafkaClientGroupId) }; } // Compute profile contains headnode, workernode, zookeepernode, edgenode, kafkamanagementnode, idbrokernode, etc. ComputeProfile computeProfile = ClusterCreateHelper.CreateComputeProfile(osProfile, vnetProfile, clusterScriptActions, ClusterType, ClusterSizeInNodes, HeadNodeSize, WorkerNodeSize, ZookeeperNodeSize, EdgeNodeSize, KafkaManagementNodeSize, EnableIDBroker.IsPresent); // Handle SecurityProfile SecurityProfile securityProfile = ClusterCreateHelper.ConvertAzureHDInsightSecurityProfileToSecurityProfile(SecurityProfile, AssignedIdentity); // Handle DisksPerWorkerNode feature Role workerNode = Utils.ExtractRole(ClusterNodeType.WorkerNode.ToString(), computeProfile); if (DisksPerWorkerNode > 0) { workerNode.DataDisksGroups = new List <DataDisksGroups>() { new DataDisksGroups() { DisksPerNode = DisksPerWorkerNode } }; } // Handle ClusterIdentity ClusterIdentity clusterIdentity = null; if (AssignedIdentity != null || StorageAccountManagedIdentity != null) { clusterIdentity = new ClusterIdentity { Type = ResourceIdentityType.UserAssigned, UserAssignedIdentities = new Dictionary <string, ClusterIdentityUserAssignedIdentitiesValue>() }; if (AssignedIdentity != null) { clusterIdentity.UserAssignedIdentities.Add(AssignedIdentity, new ClusterIdentityUserAssignedIdentitiesValue()); } if (StorageAccountManagedIdentity != null) { clusterIdentity.UserAssignedIdentities.Add(StorageAccountManagedIdentity, new ClusterIdentityUserAssignedIdentitiesValue()); } } // Handle CMK feature DiskEncryptionProperties diskEncryptionProperties = null; if (EncryptionKeyName != null && EncryptionKeyVersion != null && EncryptionVaultUri != null) { diskEncryptionProperties = new DiskEncryptionProperties() { KeyName = EncryptionKeyName, KeyVersion = EncryptionKeyVersion, VaultUri = EncryptionVaultUri, EncryptionAlgorithm = EncryptionAlgorithm != null ? EncryptionAlgorithm : JsonWebKeyEncryptionAlgorithm.RSAOAEP, MsiResourceId = AssignedIdentity }; } // Handle encryption at host feature if (EncryptionAtHost != null) { if (diskEncryptionProperties != null) { diskEncryptionProperties.EncryptionAtHost = EncryptionAtHost; } else { diskEncryptionProperties = new DiskEncryptionProperties() { EncryptionAtHost = EncryptionAtHost }; } } // Handle autoscale featurer Autoscale autoscaleParameter = null; if (AutoscaleConfiguration != null) { autoscaleParameter = AutoscaleConfiguration.ToAutoscale(); workerNode.AutoscaleConfiguration = autoscaleParameter; } // Construct cluster create parameter ClusterCreateParametersExtended createParams = new ClusterCreateParametersExtended { Location = Location, //Tags = Tags, //To Do add this Tags parameter Properties = new ClusterCreateProperties { Tier = ClusterTier, ClusterDefinition = new ClusterDefinition { Kind = ClusterType ?? "Hadoop", ComponentVersion = clusterComponentVersion, Configurations = clusterConfigurations }, ClusterVersion = Version ?? "default", KafkaRestProperties = kafkaRestProperties, ComputeProfile = computeProfile, OsType = OSType, SecurityProfile = securityProfile, StorageProfile = storageProfile, DiskEncryptionProperties = diskEncryptionProperties, //handle Encryption In Transit feature EncryptionInTransitProperties = EncryptionInTransit != null ? new EncryptionInTransitProperties() { IsEncryptionInTransitEnabled = EncryptionInTransit } : null, MinSupportedTlsVersion = MinSupportedTlsVersion }, Identity = clusterIdentity }; var cluster = HDInsightManagementClient.CreateCluster(ResourceGroupName, ClusterName, createParams); if (cluster != null) { WriteObject(new AzureHDInsightCluster(cluster)); } }
public virtual Cluster CreateCluster(string resourceGroupName, string clusterName, ClusterCreateParametersExtended createParams) { return(HdInsightManagementClient.Clusters.Create(resourceGroupName, clusterName, createParams)); }
/// <summary> /// Creates a new HDInsight cluster with the specified parameters. /// </summary> /// <param name='operations'> /// The operations group for this extension method. /// </param> /// <param name='resourceGroupName'> /// The name of the resource group. /// </param> /// <param name='clusterName'> /// The name of the cluster. /// </param> /// <param name='parameters'> /// The cluster create request. /// </param> /// <param name='cancellationToken'> /// The cancellation token. /// </param> public static async Task <Cluster> CreateAsync(this IClustersOperations operations, string resourceGroupName, string clusterName, ClusterCreateParametersExtended parameters, CancellationToken cancellationToken = default(CancellationToken)) { using (var _result = await operations.CreateWithHttpMessagesAsync(resourceGroupName, clusterName, parameters, null, cancellationToken).ConfigureAwait(false)) { return(_result.Body); } }
/// <summary> /// Creates a new HDInsight cluster with the specified parameters. /// </summary> /// <param name='operations'> /// The operations group for this extension method. /// </param> /// <param name='resourceGroupName'> /// The name of the resource group. /// </param> /// <param name='clusterName'> /// The name of the cluster. /// </param> /// <param name='parameters'> /// The cluster create request. /// </param> public static Cluster BeginCreate(this IClustersOperations operations, string resourceGroupName, string clusterName, ClusterCreateParametersExtended parameters) { return(operations.BeginCreateAsync(resourceGroupName, clusterName, parameters).GetAwaiter().GetResult()); }
internal static void ClassMain(string[] args) { // Authentication var credentials = SdkContext.AzureCredentialsFactory .FromServicePrincipal( Configurations.ClientId, Configurations.ClientSecret, Configurations.TenantId, AzureEnvironment.AzureGlobalCloud); var client = new HDInsightManagementClient(credentials) { SubscriptionId = Configurations.SubscriptionId }; string clusterName = Configurations.ClusterName; // Prepare cluster create parameters var createParams = new ClusterCreateParametersExtended { Location = Configurations.Location, Properties = new ClusterCreateProperties { ClusterVersion = "3.6", OsType = OSType.Linux, Tier = Tier.Standard, ClusterDefinition = new ClusterDefinition { Kind = "Spark", Configurations = new Dictionary <string, Dictionary <string, string> >() { { "gateway", new Dictionary <string, string> { { "restAuthCredential.isEnabled", "true" }, { "restAuthCredential.username", Configurations.ClusterLoginUserName }, { "restAuthCredential.password", Configurations.Password } } } } }, ComputeProfile = new ComputeProfile { Roles = new List <Role> { new Role { Name = "headnode", TargetInstanceCount = 2, HardwareProfile = new HardwareProfile { VmSize = "Large" }, OsProfile = new OsProfile { LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile { Username = Configurations.SshUserName, Password = Configurations.Password } } }, new Role { Name = "workernode", TargetInstanceCount = 3, HardwareProfile = new HardwareProfile { VmSize = "Large" }, OsProfile = new OsProfile { LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile { Username = Configurations.SshUserName, Password = Configurations.Password } } } } }, StorageProfile = new StorageProfile { Storageaccounts = new List <StorageAccount> { new StorageAccount { Name = Configurations.StorageAccountName + Configurations.BlobEndpointSuffix, Key = Configurations.StorageAccountKey, Container = Configurations.ContainerName.ToLowerInvariant(), IsDefault = true } } } } }; Console.WriteLine($"Start to create HDInsight Spark cluster {clusterName}"); var cluster = client.Clusters.Create(Configurations.ResourceGroupName, clusterName, createParams); Console.WriteLine($"Finish creating HDInsight Spark cluster {clusterName}"); }
internal static void ClassMain(string[] args) { // Authentication var credentials = SdkContext.AzureCredentialsFactory .FromServicePrincipal( Configurations.ClientId, Configurations.ClientSecret, Configurations.TenantId, AzureEnvironment.AzureGlobalCloud); var client = new HDInsightManagementClient(credentials) { SubscriptionId = Configurations.SubscriptionId }; string clusterName = Configurations.ClusterName; // Parse AAD-DS DNS Domain name from resource id string aaddsDnsDomainName = Configurations.AaddsResourceId.Split('/').Last(); // Prepare cluster create parameters var createParams = new ClusterCreateParametersExtended { Location = Configurations.Location, Properties = new ClusterCreateProperties { ClusterVersion = "3.6", OsType = OSType.Linux, Tier = Tier.Premium, ClusterDefinition = new ClusterDefinition { Kind = "Spark", Configurations = new Dictionary <string, Dictionary <string, string> >() { { "gateway", new Dictionary <string, string> { { "restAuthCredential.isEnabled", "true" }, { "restAuthCredential.username", Configurations.ClusterLoginUserName }, { "restAuthCredential.password", Configurations.Password } } } } }, ComputeProfile = new ComputeProfile { Roles = new List <Role> { new Role { Name = "headnode", TargetInstanceCount = 2, HardwareProfile = new HardwareProfile { VmSize = "Large" }, OsProfile = new OsProfile { LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile { Username = Configurations.SshUserName, Password = Configurations.Password } }, VirtualNetworkProfile = new VirtualNetworkProfile { Id = Configurations.VirtualNetworkResourceId, Subnet = string.Format("{0}/subnets/{1}", Configurations.VirtualNetworkResourceId, Configurations.SubnetName) } }, new Role { Name = "workernode", TargetInstanceCount = 3, HardwareProfile = new HardwareProfile { VmSize = "Large" }, OsProfile = new OsProfile { LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile { Username = Configurations.SshUserName, Password = Configurations.Password } }, VirtualNetworkProfile = new VirtualNetworkProfile { Id = Configurations.VirtualNetworkResourceId, Subnet = string.Format("{0}/subnets/{1}", Configurations.VirtualNetworkResourceId, Configurations.SubnetName) } } } }, StorageProfile = new StorageProfile { Storageaccounts = new List <StorageAccount> { new StorageAccount { Name = Configurations.StorageAccountName + Configurations.BlobEndpointSuffix, Key = Configurations.StorageAccountKey, Container = Configurations.ContainerName.ToLowerInvariant(), IsDefault = true } } }, SecurityProfile = new SecurityProfile { DirectoryType = DirectoryType.ActiveDirectory, LdapsUrls = new[] { Configurations.LdapsUrl }, DomainUsername = Configurations.DomainUserName, Domain = aaddsDnsDomainName, ClusterUsersGroupDNs = new[] { Configurations.ClusterAccessGroup }, AaddsResourceId = Configurations.AaddsResourceId, MsiResourceId = Configurations.ManagedIdentityResourceId } }, Identity = new ClusterIdentity { Type = ResourceIdentityType.UserAssigned, UserAssignedIdentities = new Dictionary <string, ClusterIdentityUserAssignedIdentitiesValue> { { Configurations.ManagedIdentityResourceId, new ClusterIdentityUserAssignedIdentitiesValue() } } } }; Console.WriteLine($"Start to create HDInsight Spark cluster {clusterName} with Enterprise Security Package"); client.Clusters.Create(Configurations.ResourceGroupName, clusterName, createParams); Console.WriteLine($"Finish creating HDInsight Spark cluster {clusterName} with Enterprise Security Package"); }
public static ClusterCreateParametersExtended GetIaasClusterSpec() { var cluster = new ClusterCreateParametersExtended { Location = "West US", Properties = new ClusterCreateProperties { ClusterDefinition = new ClusterDefinition { ClusterType = "Hadoop" }, ClusterVersion = "3.2", OperatingSystemType = OSType.Linux } }; var coreConfigs = new Dictionary <string, string> { { "fs.defaultFS", string.Format("wasb://{0}@{1}", DefaultContainer, StorageAccountName) }, { string.Format("fs.azure.account.key.{0}", StorageAccountName), StorageAccountKey } }; var gatewayConfigs = new Dictionary <string, string> { { "restAuthCredential.isEnabled", "true" }, { "restAuthCredential.username", HttpUser }, { "restAuthCredential.password", HttpPassword } }; var configurations = new Dictionary <string, Dictionary <string, string> > { { "core-site", coreConfigs }, { "gateway", gatewayConfigs } }; var serializedConfig = JsonConvert.SerializeObject(configurations); cluster.Properties.ClusterDefinition.Configurations = serializedConfig; cluster.Tags.Add("tag1", "value1"); cluster.Tags.Add("tag2", "value2"); var sshPublicKeys = new List <SshPublicKey>(); var sshPublicKey = new SshPublicKey { CertificateData = string.Format("ssh-rsa {0}", SshKey) }; sshPublicKeys.Add(sshPublicKey); var headNode = new Role { Name = "headnode", TargetInstanceCount = 1, HardwareProfile = new HardwareProfile { VmSize = "Large" }, OsProfile = new OsProfile { LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile { UserName = "******", SshProfile = new SshProfile { SshPublicKeys = sshPublicKeys } } }, VirtualNetworkProfile = new VirtualNetworkProfile { Id = "vnetid", SubnetName = "subnetname" } }; var workerNode = new Role { Name = "workernode", TargetInstanceCount = 1, HardwareProfile = new HardwareProfile { VmSize = "Large" }, OsProfile = new OsProfile { LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile { UserName = "******", SshProfile = new SshProfile { SshPublicKeys = sshPublicKeys } } } }; cluster.Properties.ComputeProfile = new ComputeProfile(); cluster.Properties.ComputeProfile.Roles.Add(headNode); cluster.Properties.ComputeProfile.Roles.Add(workerNode); return(cluster); }
public static ClusterCreateParametersExtended GetPaasClusterSpec() { var cluster = new ClusterCreateParametersExtended { Location = "East US", Properties = new ClusterCreateProperties { ClusterDefinition = new ClusterDefinition { ClusterType = "Hadoop" }, ClusterVersion = "3.1", OperatingSystemType = OSType.Windows } }; var coreConfigs = new Dictionary <string, string> { { "fs.defaultFS", string.Format("wasb://{0}@{1}", DefaultContainer, StorageAccountName) }, { string.Format("fs.azure.account.key.{0}", StorageAccountName), StorageAccountKey } }; var gatewayConfigs = new Dictionary <string, string> { { "restAuthCredential.isEnabled", "true" }, { "restAuthCredential.username", HttpUser }, { "restAuthCredential.password", HttpPassword } }; var configurations = new Dictionary <string, Dictionary <string, string> > { { "core-site", coreConfigs }, { "gateway", gatewayConfigs } }; var serializedConfig = JsonConvert.SerializeObject(configurations); cluster.Properties.ClusterDefinition.Configurations = serializedConfig; cluster.Tags.Add("tag1", "value1"); cluster.Tags.Add("tag2", "value2"); var headNode = new Role { Name = "headnode", TargetInstanceCount = 2, HardwareProfile = new HardwareProfile { VmSize = "ExtraLarge" }, OsProfile = new OsProfile { WindowsOperatingSystemProfile = new WindowsOperatingSystemProfile { RdpSettings = new RdpSettings { UserName = RdpUser, Password = RdpPassword, ExpiryDate = new DateTime(2025, 3, 1) } } } }; var workerNode = new Role { Name = "workernode", TargetInstanceCount = 5, HardwareProfile = new HardwareProfile { VmSize = "Large" } }; cluster.Properties.ComputeProfile = new ComputeProfile(); cluster.Properties.ComputeProfile.Roles.Add(headNode); cluster.Properties.ComputeProfile.Roles.Add(workerNode); return(cluster); }
public static void ValidateCluster(ClusterCreateParameters parameters, Cluster cluster, string clustername) { ClusterCreateParametersExtended extendedParams = CreateParametersConverter.GetExtendedClusterCreateParameters(clustername, parameters); ValidateCluster(extendedParams, cluster, clustername); }
public static ClusterCreateParametersExtended GetIaasClusterSpec(string containerName = DefaultContainer, bool adlStorage = false) { StorageInfo storageInfo = adlStorage ? GetDefaultDataLakeStorageInfo() : GetDefaultAzureStorageInfo(containerName); var cluster = new ClusterCreateParametersExtended { Location = Location, Tags = new Dictionary <string, string>(), Properties = new ClusterCreateProperties { ClusterVersion = "3.6", OsType = OSType.Linux, ClusterDefinition = new ClusterDefinition { Kind = "Hadoop", ComponentVersion = new Dictionary <string, string>(), Configurations = new Dictionary <string, Dictionary <string, string> >() { { "core-site", GetCoreConfigsForStorageInfo(storageInfo) }, { "gateway", new Dictionary <string, string> { { "restAuthCredential.isEnabled", "true" }, { "restAuthCredential.username", "admin" }, { "restAuthCredential.password", "Password1!" } } } } }, Tier = Tier.Standard, ComputeProfile = new ComputeProfile { Roles = new List <Role> { new Role { Name = "headnode", TargetInstanceCount = 2, HardwareProfile = new HardwareProfile { VmSize = DefaultVmSizes.HeadNode.GetSize("Hadoop") }, OsProfile = new OsProfile { LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile { Username = SshUser, Password = SshPassword } } }, new Role { Name = "workernode", TargetInstanceCount = 3, HardwareProfile = new HardwareProfile { VmSize = "Large" }, OsProfile = new OsProfile { LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile { Username = SshUser, Password = SshPassword } } }, new Role { Name = "zookeepernode", TargetInstanceCount = 3, HardwareProfile = new HardwareProfile { VmSize = DefaultVmSizes.ZookeeperNode.GetSize("Hadoop") }, OsProfile = new OsProfile { LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile { Username = SshUser, Password = SshPassword } } } } } } }; return(cluster); }
/// <summary> /// Creates a new HDInsight cluster with the specified parameters. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.HDInsight.IClusterOperations. /// </param> /// <param name='resourceGroupName'> /// Required. The name of the resource group. /// </param> /// <param name='clusterName'> /// Required. The name of the cluster. /// </param> /// <param name='clusterCreateParameters'> /// Required. The cluster create request. /// </param> /// <returns> /// The GetCluster operation response. /// </returns> public static ClusterGetResponse Create(this IClusterOperations operations, string resourceGroupName, string clusterName, ClusterCreateParametersExtended clusterCreateParameters) { return(Task.Factory.StartNew((object s) => { return ((IClusterOperations)s).CreateAsync(resourceGroupName, clusterName, clusterCreateParameters); } , operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult()); }
/// <summary> /// Creates a new HDInsight cluster with the specified parameters. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.HDInsight.IClusterOperations. /// </param> /// <param name='resourceGroupName'> /// Required. The name of the resource group. /// </param> /// <param name='clusterName'> /// Required. The name of the cluster. /// </param> /// <param name='clusterCreateParameters'> /// Required. The cluster create request. /// </param> /// <returns> /// The GetCluster operation response. /// </returns> public static Task <ClusterGetResponse> CreateAsync(this IClusterOperations operations, string resourceGroupName, string clusterName, ClusterCreateParametersExtended clusterCreateParameters) { return(operations.CreateAsync(resourceGroupName, clusterName, clusterCreateParameters, CancellationToken.None)); }
/// <summary> /// Create cluster create parameters for ADLS Gen1 relevant tests. /// </summary> /// <param name="commonData"></param> /// <param name="createParams"> /// If provided, the method will update the given parameters; /// Otherwise, a new create parameters will be created. /// </param> /// <returns></returns> public static ClusterCreateParametersExtended PrepareClusterCreateParamsForADLSv1(this CommonTestFixture commonData, ClusterCreateParametersExtended createParams = null) { var createParamsForADLSv1 = createParams ?? commonData.PrepareClusterCreateParams(); var configurations = (Dictionary <string, Dictionary <string, string> >)createParamsForADLSv1.Properties.ClusterDefinition.Configurations; string clusterIdentity = "clusterIdentity"; var clusterIdentityConfig = new Dictionary <string, string>() { { "clusterIdentity.applicationId", commonData.DataLakeClientId }, { "clusterIdentity.certificate", commonData.CertContent }, { "clusterIdentity.aadTenantId", "https://login.windows.net/" + commonData.TenantId }, { "clusterIdentity.resourceUri", "https://datalake.azure.net/" }, { "clusterIdentity.certificatePassword", commonData.CertPassword } }; configurations.Add(clusterIdentity, clusterIdentityConfig); bool isDefault = !createParamsForADLSv1.Properties.StorageProfile.Storageaccounts.Any(); if (isDefault) { string coreSite = "core-site"; var coreConfig = new Dictionary <string, string>() { { "fs.defaultFS", "adl://home" }, { "dfs.adls.home.hostname", commonData.DataLakeStoreAccountName + ".azuredatalakestore.net" }, { "dfs.adls.home.mountpoint", commonData.DataLakeStoreMountpoint } }; configurations.Add(coreSite, coreConfig); } return(createParamsForADLSv1); }
internal static void ClassMain(string[] args) { // Authentication var credentials = SdkContext.AzureCredentialsFactory .FromServicePrincipal( Configurations.ClientId, Configurations.ClientSecret, Configurations.TenantId, AzureEnvironment.AzureGlobalCloud); var client = new HDInsightManagementClient(credentials) { SubscriptionId = Configurations.SubscriptionId }; // Parse ADLS Gen2 storage account name from resource id string adlsGen2AccountName = Configurations.AdlsGen2ResourceId.Split('/').Last(); string clusterName = Configurations.ClusterName; // Prepare cluster create parameters var createParamsForADLSv2 = new ClusterCreateParametersExtended { Location = Configurations.Location, Properties = new ClusterCreateProperties { ClusterVersion = "3.6", OsType = OSType.Linux, Tier = Tier.Standard, ClusterDefinition = new ClusterDefinition { Kind = "Hadoop", Configurations = new Dictionary <string, Dictionary <string, string> >() { { "gateway", new Dictionary <string, string> { { "restAuthCredential.isEnabled", "true" }, { "restAuthCredential.username", Configurations.ClusterLoginUserName }, { "restAuthCredential.password", Configurations.Password } } } } }, ComputeProfile = new ComputeProfile { Roles = new List <Role> { new Role { Name = "headnode", TargetInstanceCount = 2, HardwareProfile = new HardwareProfile { VmSize = "Large" }, OsProfile = new OsProfile { LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile { Username = Configurations.SshUserName, Password = Configurations.Password } } }, new Role { Name = "workernode", TargetInstanceCount = 3, HardwareProfile = new HardwareProfile { VmSize = "Large" }, OsProfile = new OsProfile { LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile { Username = Configurations.SshUserName, Password = Configurations.Password } } } } }, StorageProfile = new StorageProfile { Storageaccounts = new List <StorageAccount> { new StorageAccount { Name = adlsGen2AccountName + Configurations.DfsEndpointSuffix, IsDefault = true, FileSystem = Configurations.AdlsGen2FileSystemName.ToLowerInvariant(), ResourceId = Configurations.AdlsGen2ResourceId, MsiResourceId = Configurations.ManagedIdentityResourceId } } } }, Identity = new ClusterIdentity { Type = ResourceIdentityType.UserAssigned, UserAssignedIdentities = new Dictionary <string, ClusterIdentityUserAssignedIdentitiesValue> { { Configurations.ManagedIdentityResourceId, new ClusterIdentityUserAssignedIdentitiesValue() } } } }; Console.WriteLine($"Starting to create HDInsight Hadoop cluster {clusterName} with Azure Data Lake Storage Gen2"); client.Clusters.Create(Configurations.ResourceGroupName, clusterName, createParamsForADLSv2); Console.WriteLine($"Finished creating HDInsight Hadoop cluster {clusterName} with Azure Data Lake Storage Gen2"); }
private ClusterCreateParametersExtended GetExtendedClusterCreateParameters( string clusterName, ClusterCreateParameters clusterCreateParameters) { var createParamsExtended = new ClusterCreateParametersExtended { Location = clusterCreateParameters.Location, Properties = new ClusterCreateProperties { ClusterDefinition = new ClusterDefinition { ClusterType = clusterCreateParameters.ClusterType }, ClusterVersion = clusterCreateParameters.Version, OperatingSystemType = clusterCreateParameters.OSType, ClusterTier = clusterCreateParameters.ClusterTier } }; var configurations = GetConfigurations(clusterName, clusterCreateParameters); if (clusterCreateParameters.HiveMetastore != null) { var metastoreConfig = GetMetastoreConfig(clusterCreateParameters.HiveMetastore, clusterCreateParameters.OSType, "Hive"); foreach (var configSet in metastoreConfig) { if (configurations.ContainsKey(configSet.Key)) { foreach (var config in configSet.Value) { configurations[configSet.Key].Add(config.Key, config.Value); } } else { configurations.Add(configSet.Key, configSet.Value); } } } if (clusterCreateParameters.OozieMetastore != null) { var metastoreConfig = GetMetastoreConfig(clusterCreateParameters.OozieMetastore, clusterCreateParameters.OSType, "oozie"); foreach (var configSet in metastoreConfig) { if (configurations.ContainsKey(configSet.Key)) { foreach (var config in configSet.Value) { configurations[configSet.Key].Add(config.Key, config.Value); } } else { configurations.Add(configSet.Key, configSet.Value); } } } var serializedConfig = JsonConvert.SerializeObject(configurations); createParamsExtended.Properties.ClusterDefinition.Configurations = serializedConfig; var roles = GetRoleCollection(clusterCreateParameters); createParamsExtended.Properties.ComputeProfile = new ComputeProfile(); foreach (var role in roles) { createParamsExtended.Properties.ComputeProfile.Roles.Add(role); } return(createParamsExtended); }
public void ConvertParametersWithMissingNoProvidedPropertiesShouldNotThrow() { ClusterCreateParameters createParams = new ClusterCreateParameters(); ClusterCreateParametersExtended extendedParams = CreateParametersConverter.GetExtendedClusterCreateParameters("testCluster", createParams); }