public static string GetClusterPasswordFromPayloadObject(
Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters cluster)
{
GatewayComponent gateway = cluster.Components.OfType <GatewayComponent>().Single();
return(gateway.RestAuthCredential.Password);
}
public static int GetClusterSizeFromPayloadObject(Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters cluster)
{
var workerNodeRoles = cluster.ClusterRoleCollection.ToList().Where(role => role.FriendlyName == WorkerNodeRoleName).ToList();
if (workerNodeRoles.Any())
{
return(workerNodeRoles.First().InstanceCount);
}
return(0);
}
public static IEnumerable <WabStorageAccountConfiguration> GetAdditionalStorageAccountFromFromPayloadObject(
Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters cluster)
{
// For Yarn clusters (HDI version starts from 3.0), the additional storage account is for MapReduceApplication
YarnComponent yarn = cluster.Components.OfType <YarnComponent>().Single();
MapReduceApplication mrApp = yarn.Applications.OfType <MapReduceApplication>().Single();
var additionalStorageAccounts = mrApp.AdditionalStorageContainers.ToList();
if (additionalStorageAccounts.Any())
{
var result = (from BlobContainerCredentialBackedResource tem in additionalStorageAccounts
select new WabStorageAccountConfiguration(tem.AccountDnsName, tem.Key, tem.BlobContainerName)).ToList();
return(result);
}
return(Enumerable.Empty <WabStorageAccountConfiguration>());
}
public static WabStorageAccountConfiguration GetDefaultStorageAccountFromFromPayloadObject(
Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters cluster)
{
// For Yarn clusters (version later than 3.0 inclusive), the default storage account is for MapReduceApplication
YarnComponent yarn = cluster.Components.OfType <YarnComponent>().Single();
MapReduceApplication mrApp = yarn.Applications.OfType <MapReduceApplication>().Single();
if (mrApp.DefaultStorageAccountAndContainer.ShouldProvisionNew)
{
return(null);
}
return(new WabStorageAccountConfiguration(
mrApp.DefaultStorageAccountAndContainer.AccountDnsName,
mrApp.DefaultStorageAccountAndContainer.Key,
mrApp.DefaultStorageAccountAndContainer.BlobContainerName));
}
private static HDInsight.ClusterCreateParametersV2 CreateClusterRequest_FromInternalV3(
Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters payloadObject)
{
var cluster = new HDInsight.ClusterCreateParametersV2
{
Location = payloadObject.Location,
Name = payloadObject.DnsName,
UserName = GetClusterUsernameFromPayloadObject(payloadObject),
Password = GetClusterPasswordFromPayloadObject(payloadObject),
Version = payloadObject.Version,
DefaultStorageAccountName = GetDefaultStorageAccountFromFromPayloadObject(payloadObject).Name,
DefaultStorageAccountKey = GetDefaultStorageAccountFromFromPayloadObject(payloadObject).Key,
DefaultStorageContainer = GetDefaultStorageAccountFromFromPayloadObject(payloadObject).Container,
ClusterSizeInNodes = payloadObject.ClusterRoleCollection.ToList().Single(role => role.FriendlyName == WorkerNodeRoleName).InstanceCount,
};
var headNodeRole = payloadObject.ClusterRoleCollection.ToList().Where(role => role.FriendlyName == HeadNodeRoleName).ToList();
if (headNodeRole.Any())
{
cluster.HeadNodeSize = headNodeRole.First().VMSizeAsString;
}
var dataNodeRole = payloadObject.ClusterRoleCollection.ToList().Where(role => role.FriendlyName == WorkerNodeRoleName).ToList();
if (dataNodeRole.Any())
{
cluster.DataNodeSize = dataNodeRole.First().VMSizeAsString;
}
var zookeeperNodeRole = payloadObject.ClusterRoleCollection.ToList().Where(role => role.FriendlyName == ZookeeperNodeRoleName).ToList();
if (zookeeperNodeRole.Any())
{
cluster.ZookeeperNodeSize = zookeeperNodeRole.First().VMSizeAsString;
}
if (payloadObject.VirtualNetworkConfiguration != null)
{
cluster.VirtualNetworkId = payloadObject.VirtualNetworkConfiguration.VirtualNetworkSite;
cluster.SubnetName = payloadObject.VirtualNetworkConfiguration.AddressAssignments.First().Subnets.First().Name;
}
CopyConfigurationForCluster(payloadObject, cluster);
return(cluster);
}
private static void CopyConfigurationForCluster(
Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters payloadObject, HDInsight.ClusterCreateParameters cluster)
{
var yarn = payloadObject.Components.OfType <YarnComponent>().Single();
var mapreduce = yarn.Applications.OfType <MapReduceApplication>().Single();
var hive = payloadObject.Components.OfType <HiveComponent>().Single();
var oozie = payloadObject.Components.OfType <OozieComponent>().Single();
var hdfs = payloadObject.Components.OfType <HdfsComponent>().Single();
var hadoopCore = payloadObject.Components.OfType <HadoopCoreComponent>().Single();
HBaseComponent hbase = null;
if (payloadObject.Components.OfType <HBaseComponent>().Count() == 1)
{
hbase = payloadObject.Components.OfType <HBaseComponent>().Single();
}
StormComponent storm = null;
if (payloadObject.Components.OfType <StormComponent>().Count() == 1)
{
storm = payloadObject.Components.OfType <StormComponent>().Single();
}
CustomActionComponent configActions = null;
if (payloadObject.Components.OfType <CustomActionComponent>().Count() == 1)
{
configActions = payloadObject.Components.OfType <CustomActionComponent>().Single();
}
if (hadoopCore.CoreSiteXmlProperties.Any())
{
cluster.CoreConfiguration.AddRange(
hadoopCore.CoreSiteXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (hdfs.HdfsSiteXmlProperties.Any())
{
cluster.HdfsConfiguration.AddRange(hdfs.HdfsSiteXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (mapreduce.MapRedSiteXmlProperties.Any())
{
cluster.MapReduceConfiguration.ConfigurationCollection.AddRange(
mapreduce.MapRedSiteXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (mapreduce.CapacitySchedulerConfiguration.Any())
{
cluster.MapReduceConfiguration.CapacitySchedulerConfigurationCollection.AddRange(
mapreduce.CapacitySchedulerConfiguration.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (mapreduce.AdditionalStorageContainers.ToList().Any())
{
cluster.AdditionalStorageAccounts.AddRange(
from BlobContainerCredentialBackedResource tem in mapreduce.AdditionalStorageContainers
select new WabStorageAccountConfiguration(tem.AccountDnsName, tem.Key, tem.BlobContainerName));
}
if (yarn.Configuration.Any())
{
cluster.YarnConfiguration.AddRange(yarn.Configuration.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (hive.HiveSiteXmlProperties.Any())
{
cluster.HiveConfiguration.ConfigurationCollection.AddRange(
hive.HiveSiteXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (hive.AdditionalLibraries != null)
{
cluster.HiveConfiguration.AdditionalLibraries = new WabStorageAccountConfiguration(
hive.AdditionalLibraries.AccountDnsName, hive.AdditionalLibraries.Key, hive.AdditionalLibraries.BlobContainerName);
}
if (!hive.Metastore.ShouldProvisionNew)
{
var metaStore = (SqlAzureDatabaseCredentialBackedResource)hive.Metastore;
cluster.HiveMetastore = new Metastore(
metaStore.SqlServerName, metaStore.DatabaseName, metaStore.Credentials.Username, metaStore.Credentials.Password);
}
if (configActions != null)
{
foreach (var configAction in configActions.CustomActions)
{
ScriptCustomAction sca = configAction as ScriptCustomAction;
if (sca != null)
{
cluster.ConfigActions.Add(new ScriptAction(
sca.Name, ConvertClusterRoleToClusterNodeType(sca), sca.Uri, sca.Parameters));
}
}
}
if (oozie.Configuration.Any())
{
cluster.OozieConfiguration.ConfigurationCollection.AddRange(
oozie.Configuration.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (oozie.AdditionalSharedLibraries != null)
{
cluster.OozieConfiguration.AdditionalSharedLibraries =
new WabStorageAccountConfiguration(
oozie.AdditionalSharedLibraries.AccountDnsName,
oozie.AdditionalSharedLibraries.Key,
oozie.AdditionalSharedLibraries.BlobContainerName);
}
if (oozie.AdditionalActionExecutorLibraries != null)
{
cluster.OozieConfiguration.AdditionalActionExecutorLibraries =
new WabStorageAccountConfiguration(
oozie.AdditionalActionExecutorLibraries.AccountDnsName,
oozie.AdditionalActionExecutorLibraries.Key,
oozie.AdditionalActionExecutorLibraries.BlobContainerName);
}
if (!oozie.Metastore.ShouldProvisionNew)
{
var metaStore = (SqlAzureDatabaseCredentialBackedResource)oozie.Metastore;
cluster.OozieMetastore = new Metastore(
metaStore.SqlServerName, metaStore.DatabaseName, metaStore.Credentials.Username, metaStore.Credentials.Password);
}
if (hbase != null && hbase.HBaseConfXmlProperties.Any())
{
cluster.HBaseConfiguration.ConfigurationCollection.AddRange(
hbase.HBaseConfXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (hbase != null && hbase.AdditionalLibraries != null)
{
cluster.HBaseConfiguration.AdditionalLibraries = new WabStorageAccountConfiguration(
hbase.AdditionalLibraries.AccountDnsName, hbase.AdditionalLibraries.Key, hbase.AdditionalLibraries.BlobContainerName);
}
if (storm != null && storm.StormConfiguration.Any())
{
cluster.StormConfiguration.AddRange(
storm.StormConfiguration.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
}