public static string GetClusterPasswordFromPayloadObject( Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters cluster) { GatewayComponent gateway = cluster.Components.OfType <GatewayComponent>().Single(); return(gateway.RestAuthCredential.Password); }
public static int GetClusterSizeFromPayloadObject(Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters cluster) { var workerNodeRoles = cluster.ClusterRoleCollection.ToList().Where(role => role.FriendlyName == WorkerNodeRoleName).ToList(); if (workerNodeRoles.Any()) { return(workerNodeRoles.First().InstanceCount); } return(0); }
public static IEnumerable <WabStorageAccountConfiguration> GetAdditionalStorageAccountFromFromPayloadObject( Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters cluster) { // For Yarn clusters (HDI version starts from 3.0), the additional storage account is for MapReduceApplication YarnComponent yarn = cluster.Components.OfType <YarnComponent>().Single(); MapReduceApplication mrApp = yarn.Applications.OfType <MapReduceApplication>().Single(); var additionalStorageAccounts = mrApp.AdditionalStorageContainers.ToList(); if (additionalStorageAccounts.Any()) { var result = (from BlobContainerCredentialBackedResource tem in additionalStorageAccounts select new WabStorageAccountConfiguration(tem.AccountDnsName, tem.Key, tem.BlobContainerName)).ToList(); return(result); } return(Enumerable.Empty <WabStorageAccountConfiguration>()); }
public static WabStorageAccountConfiguration GetDefaultStorageAccountFromFromPayloadObject( Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters cluster) { // For Yarn clusters (version later than 3.0 inclusive), the default storage account is for MapReduceApplication YarnComponent yarn = cluster.Components.OfType <YarnComponent>().Single(); MapReduceApplication mrApp = yarn.Applications.OfType <MapReduceApplication>().Single(); if (mrApp.DefaultStorageAccountAndContainer.ShouldProvisionNew) { return(null); } return(new WabStorageAccountConfiguration( mrApp.DefaultStorageAccountAndContainer.AccountDnsName, mrApp.DefaultStorageAccountAndContainer.Key, mrApp.DefaultStorageAccountAndContainer.BlobContainerName)); }
private static HDInsight.ClusterCreateParametersV2 CreateClusterRequest_FromInternalV3( Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters payloadObject) { var cluster = new HDInsight.ClusterCreateParametersV2 { Location = payloadObject.Location, Name = payloadObject.DnsName, UserName = GetClusterUsernameFromPayloadObject(payloadObject), Password = GetClusterPasswordFromPayloadObject(payloadObject), Version = payloadObject.Version, DefaultStorageAccountName = GetDefaultStorageAccountFromFromPayloadObject(payloadObject).Name, DefaultStorageAccountKey = GetDefaultStorageAccountFromFromPayloadObject(payloadObject).Key, DefaultStorageContainer = GetDefaultStorageAccountFromFromPayloadObject(payloadObject).Container, ClusterSizeInNodes = payloadObject.ClusterRoleCollection.ToList().Single(role => role.FriendlyName == WorkerNodeRoleName).InstanceCount, }; var headNodeRole = payloadObject.ClusterRoleCollection.ToList().Where(role => role.FriendlyName == HeadNodeRoleName).ToList(); if (headNodeRole.Any()) { cluster.HeadNodeSize = headNodeRole.First().VMSizeAsString; } var dataNodeRole = payloadObject.ClusterRoleCollection.ToList().Where(role => role.FriendlyName == WorkerNodeRoleName).ToList(); if (dataNodeRole.Any()) { cluster.DataNodeSize = dataNodeRole.First().VMSizeAsString; } var zookeeperNodeRole = payloadObject.ClusterRoleCollection.ToList().Where(role => role.FriendlyName == ZookeeperNodeRoleName).ToList(); if (zookeeperNodeRole.Any()) { cluster.ZookeeperNodeSize = zookeeperNodeRole.First().VMSizeAsString; } if (payloadObject.VirtualNetworkConfiguration != null) { cluster.VirtualNetworkId = payloadObject.VirtualNetworkConfiguration.VirtualNetworkSite; cluster.SubnetName = payloadObject.VirtualNetworkConfiguration.AddressAssignments.First().Subnets.First().Name; } CopyConfigurationForCluster(payloadObject, cluster); return(cluster); }
private static void CopyConfigurationForCluster( Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters payloadObject, HDInsight.ClusterCreateParameters cluster) { var yarn = payloadObject.Components.OfType <YarnComponent>().Single(); var mapreduce = yarn.Applications.OfType <MapReduceApplication>().Single(); var hive = payloadObject.Components.OfType <HiveComponent>().Single(); var oozie = payloadObject.Components.OfType <OozieComponent>().Single(); var hdfs = payloadObject.Components.OfType <HdfsComponent>().Single(); var hadoopCore = payloadObject.Components.OfType <HadoopCoreComponent>().Single(); HBaseComponent hbase = null; if (payloadObject.Components.OfType <HBaseComponent>().Count() == 1) { hbase = payloadObject.Components.OfType <HBaseComponent>().Single(); } StormComponent storm = null; if (payloadObject.Components.OfType <StormComponent>().Count() == 1) { storm = payloadObject.Components.OfType <StormComponent>().Single(); } CustomActionComponent configActions = null; if (payloadObject.Components.OfType <CustomActionComponent>().Count() == 1) { configActions = payloadObject.Components.OfType <CustomActionComponent>().Single(); } if (hadoopCore.CoreSiteXmlProperties.Any()) { cluster.CoreConfiguration.AddRange( hadoopCore.CoreSiteXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value))); } if (hdfs.HdfsSiteXmlProperties.Any()) { cluster.HdfsConfiguration.AddRange(hdfs.HdfsSiteXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value))); } if (mapreduce.MapRedSiteXmlProperties.Any()) { cluster.MapReduceConfiguration.ConfigurationCollection.AddRange( mapreduce.MapRedSiteXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value))); } if (mapreduce.CapacitySchedulerConfiguration.Any()) { cluster.MapReduceConfiguration.CapacitySchedulerConfigurationCollection.AddRange( mapreduce.CapacitySchedulerConfiguration.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value))); } if (mapreduce.AdditionalStorageContainers.ToList().Any()) { cluster.AdditionalStorageAccounts.AddRange( from BlobContainerCredentialBackedResource tem in mapreduce.AdditionalStorageContainers select new WabStorageAccountConfiguration(tem.AccountDnsName, tem.Key, tem.BlobContainerName)); } if (yarn.Configuration.Any()) { cluster.YarnConfiguration.AddRange(yarn.Configuration.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value))); } if (hive.HiveSiteXmlProperties.Any()) { cluster.HiveConfiguration.ConfigurationCollection.AddRange( hive.HiveSiteXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value))); } if (hive.AdditionalLibraries != null) { cluster.HiveConfiguration.AdditionalLibraries = new WabStorageAccountConfiguration( hive.AdditionalLibraries.AccountDnsName, hive.AdditionalLibraries.Key, hive.AdditionalLibraries.BlobContainerName); } if (!hive.Metastore.ShouldProvisionNew) { var metaStore = (SqlAzureDatabaseCredentialBackedResource)hive.Metastore; cluster.HiveMetastore = new Metastore( metaStore.SqlServerName, metaStore.DatabaseName, metaStore.Credentials.Username, metaStore.Credentials.Password); } if (configActions != null) { foreach (var configAction in configActions.CustomActions) { ScriptCustomAction sca = configAction as ScriptCustomAction; if (sca != null) { cluster.ConfigActions.Add(new ScriptAction( sca.Name, ConvertClusterRoleToClusterNodeType(sca), sca.Uri, sca.Parameters)); } } } if (oozie.Configuration.Any()) { cluster.OozieConfiguration.ConfigurationCollection.AddRange( oozie.Configuration.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value))); } if (oozie.AdditionalSharedLibraries != null) { cluster.OozieConfiguration.AdditionalSharedLibraries = new WabStorageAccountConfiguration( oozie.AdditionalSharedLibraries.AccountDnsName, oozie.AdditionalSharedLibraries.Key, oozie.AdditionalSharedLibraries.BlobContainerName); } if (oozie.AdditionalActionExecutorLibraries != null) { cluster.OozieConfiguration.AdditionalActionExecutorLibraries = new WabStorageAccountConfiguration( oozie.AdditionalActionExecutorLibraries.AccountDnsName, oozie.AdditionalActionExecutorLibraries.Key, oozie.AdditionalActionExecutorLibraries.BlobContainerName); } if (!oozie.Metastore.ShouldProvisionNew) { var metaStore = (SqlAzureDatabaseCredentialBackedResource)oozie.Metastore; cluster.OozieMetastore = new Metastore( metaStore.SqlServerName, metaStore.DatabaseName, metaStore.Credentials.Username, metaStore.Credentials.Password); } if (hbase != null && hbase.HBaseConfXmlProperties.Any()) { cluster.HBaseConfiguration.ConfigurationCollection.AddRange( hbase.HBaseConfXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value))); } if (hbase != null && hbase.AdditionalLibraries != null) { cluster.HBaseConfiguration.AdditionalLibraries = new WabStorageAccountConfiguration( hbase.AdditionalLibraries.AccountDnsName, hbase.AdditionalLibraries.Key, hbase.AdditionalLibraries.BlobContainerName); } if (storm != null && storm.StormConfiguration.Any()) { cluster.StormConfiguration.AddRange( storm.StormConfiguration.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value))); } }