private void AssertContainerMatchesNameProvided(ClusterCreateParametersExtended createParamsExtended, string name) { Dictionary<string,string> coresiteConfig; if (JsonConvert.DeserializeObject<Dictionary<string, Dictionary<string, string>>>(createParamsExtended.Properties.ClusterDefinition.Configurations).TryGetValue("core-site", out coresiteConfig)) { string value; coresiteConfig.TryGetValue("fs.defaultFS", out value); Assert.True(value.StartsWith("wasb://" + name + "@"), "Container does not match the name provided"); } else { Assert.True(false, "Deserialization error"); } }
private ClusterCreateParametersExtended GetExtendedClusterCreateParameters( string clusterName, ClusterCreateParameters clusterCreateParameters) { var createParamsExtended = new ClusterCreateParametersExtended { Location = clusterCreateParameters.Location, Properties = new ClusterCreateProperties { ClusterDefinition = new ClusterDefinition { ClusterType = clusterCreateParameters.ClusterType }, ClusterVersion = clusterCreateParameters.Version, OperatingSystemType = clusterCreateParameters.OSType } }; var configurations = GetConfigurations(clusterName, clusterCreateParameters); if (clusterCreateParameters.HiveMetastore != null) { var metastoreConfig = GetMetastoreConfig(clusterCreateParameters.HiveMetastore, clusterCreateParameters.OSType, "Hive"); foreach (var configSet in metastoreConfig) { if (configurations.ContainsKey(configSet.Key)) { foreach (var config in configSet.Value) { configurations[configSet.Key].Add(config.Key, config.Value); } } else { configurations.Add(configSet.Key, configSet.Value); } } } if (clusterCreateParameters.OozieMetastore != null) { var metastoreConfig = GetMetastoreConfig(clusterCreateParameters.OozieMetastore, clusterCreateParameters.OSType, "oozie"); foreach (var configSet in metastoreConfig) { if (configurations.ContainsKey(configSet.Key)) { foreach (var config in configSet.Value) { configurations[configSet.Key].Add(config.Key, config.Value); } } else { configurations.Add(configSet.Key, configSet.Value); } } } var serializedConfig = JsonConvert.SerializeObject(configurations); createParamsExtended.Properties.ClusterDefinition.Configurations = serializedConfig; var roles = GetRoleCollection(clusterCreateParameters); createParamsExtended.Properties.ComputeProfile = new ComputeProfile(); foreach (var role in roles) { role.SecurityProfile = clusterCreateParameters.SecurityProfile; createParamsExtended.Properties.ComputeProfile.Roles.Add(role); } return createParamsExtended; }
public static ClusterCreateParametersExtended GetIaasClusterSpec() { var cluster = new ClusterCreateParametersExtended { Location = "West US", Properties = new ClusterCreateProperties { ClusterDefinition = new ClusterDefinition { ClusterType = "Hadoop" }, ClusterVersion = "3.2", OperatingSystemType = OSType.Linux } }; var coreConfigs = new Dictionary<string, string> { {"fs.defaultFS", string.Format("wasb://{0}@{1}", DefaultContainer, StorageAccountName)}, { string.Format("fs.azure.account.key.{0}", StorageAccountName), StorageAccountKey } }; var gatewayConfigs = new Dictionary<string, string> { {"restAuthCredential.isEnabled", "true"}, {"restAuthCredential.username", HttpUser}, {"restAuthCredential.password", HttpPassword} }; var configurations = new Dictionary<string, Dictionary<string, string>> { {"core-site", coreConfigs}, {"gateway", gatewayConfigs} }; var serializedConfig = JsonConvert.SerializeObject(configurations); cluster.Properties.ClusterDefinition.Configurations = serializedConfig; cluster.Tags.Add("tag1", "value1"); cluster.Tags.Add("tag2", "value2"); var sshPublicKeys = new List<SshPublicKey>(); var sshPublicKey = new SshPublicKey { CertificateData = string.Format("ssh-rsa {0}", SshKey) }; sshPublicKeys.Add(sshPublicKey); var headNode = new Role { Name = "headnode", TargetInstanceCount = 1, HardwareProfile = new HardwareProfile { VmSize = "Large" }, OsProfile = new OsProfile { LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile { UserName = "******", SshProfile = new SshProfile { SshPublicKeys = sshPublicKeys } } }, VirtualNetworkProfile = new VirtualNetworkProfile { Id = "vnetid", SubnetName = "subnetname" } }; var workerNode = new Role { Name = "workernode", TargetInstanceCount = 1, HardwareProfile = new HardwareProfile { VmSize = "Large" }, OsProfile = new OsProfile { LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile { UserName = "******", SshProfile = new SshProfile { SshPublicKeys = sshPublicKeys } } } }; cluster.Properties.ComputeProfile = new ComputeProfile(); cluster.Properties.ComputeProfile.Roles.Add(headNode); cluster.Properties.ComputeProfile.Roles.Add(workerNode); return cluster; }
public static ClusterCreateParametersExtended GetPaasClusterSpec() { var cluster = new ClusterCreateParametersExtended { Location = "West US", Properties = new ClusterCreateProperties { ClusterDefinition = new ClusterDefinition { ClusterType = "Hadoop" }, ClusterVersion = "3.1", OperatingSystemType = OSType.Windows } }; var coreConfigs = new Dictionary<string, string> { {"fs.defaultFS", string.Format("wasb://{0}@{1}", DefaultContainer, StorageAccountName)}, { string.Format("fs.azure.account.key.{0}", StorageAccountName), StorageAccountKey } }; var gatewayConfigs = new Dictionary<string, string> { {"restAuthCredential.isEnabled", "true"}, {"restAuthCredential.username", HttpUser}, {"restAuthCredential.password", HttpPassword} }; var configurations = new Dictionary<string, Dictionary<string, string>> { {"core-site", coreConfigs}, {"gateway", gatewayConfigs} }; var serializedConfig = JsonConvert.SerializeObject(configurations); cluster.Properties.ClusterDefinition.Configurations = serializedConfig; cluster.Tags.Add("tag1", "value1"); cluster.Tags.Add("tag2", "value2"); var headNode = new Role { Name = "headnode", TargetInstanceCount = 2, HardwareProfile = new HardwareProfile { VmSize = "ExtraLarge" }, OsProfile = new OsProfile { WindowsOperatingSystemProfile = new WindowsOperatingSystemProfile { RdpSettings = new RdpSettings { UserName = RdpUser, Password = RdpPassword, ExpiryDate = new DateTime(2025, 3, 1) } } } }; var workerNode = new Role { Name = "workernode", TargetInstanceCount = 5, HardwareProfile = new HardwareProfile { VmSize = "Large" } }; cluster.Properties.ComputeProfile = new ComputeProfile(); cluster.Properties.ComputeProfile.Roles.Add(headNode); cluster.Properties.ComputeProfile.Roles.Add(workerNode); return cluster; }
public static ClusterCreateParametersExtended AddConfigurations(ClusterCreateParametersExtended cluster, string configurationKey, Dictionary<string, string> configs) { string configurations = cluster.Properties.ClusterDefinition.Configurations; var config = JsonConvert.DeserializeObject<Dictionary<string, Dictionary<String, string>>>(configurations); config.Add(configurationKey, configs); var serializedConfig = JsonConvert.SerializeObject(config); cluster.Properties.ClusterDefinition.Configurations = serializedConfig; return cluster; }
/// <summary> /// Creates a new HDInsight cluster with the specified parameters. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.HDInsight.IClusterOperations. /// </param> /// <param name='resourceGroupName'> /// Required. The name of the resource group. /// </param> /// <param name='clusterName'> /// Required. The name of the cluster. /// </param> /// <param name='clusterCreateParameters'> /// Required. The cluster create request. /// </param> /// <returns> /// The GetCluster operation response. /// </returns> public static Task<ClusterGetResponse> CreateAsync(this IClusterOperations operations, string resourceGroupName, string clusterName, ClusterCreateParametersExtended clusterCreateParameters) { return operations.CreateAsync(resourceGroupName, clusterName, clusterCreateParameters, CancellationToken.None); }
/// <summary> /// Creates a new HDInsight cluster with the specified parameters. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.HDInsight.IClusterOperations. /// </param> /// <param name='resourceGroupName'> /// Required. The name of the resource group. /// </param> /// <param name='clusterName'> /// Required. The name of the cluster. /// </param> /// <param name='clusterCreateParameters'> /// Required. The cluster create request. /// </param> /// <returns> /// The GetCluster operation response. /// </returns> public static ClusterGetResponse Create(this IClusterOperations operations, string resourceGroupName, string clusterName, ClusterCreateParametersExtended clusterCreateParameters) { return Task.Factory.StartNew((object s) => { return ((IClusterOperations)s).CreateAsync(resourceGroupName, clusterName, clusterCreateParameters); } , operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult(); }