private static void ConfigHiveComponent(HiveComponent hive, HDInsight.ClusterCreateParametersV2 inputs)
{
hive.HiveSiteXmlProperties.AddRange(
inputs.HiveConfiguration.ConfigurationCollection.Select(prop => new Property {
Name = prop.Key, Value = prop.Value
}));
if (inputs.HiveConfiguration.AdditionalLibraries != null)
{
hive.AdditionalLibraries = new BlobContainerCredentialBackedResource()
{
AccountDnsName = inputs.HiveConfiguration.AdditionalLibraries.Name,
BlobContainerName = inputs.HiveConfiguration.AdditionalLibraries.Container,
Key = inputs.HiveConfiguration.AdditionalLibraries.Key
};
}
if (inputs.HiveMetastore != null)
{
hive.Metastore = new SqlAzureDatabaseCredentialBackedResource()
{
SqlServerName = inputs.HiveMetastore.Server,
Credentials = new UsernamePasswordCredential()
{
Username = inputs.HiveMetastore.User, Password = inputs.HiveMetastore.Password
},
DatabaseName = inputs.HiveMetastore.Database
};
}
}
/// <summary>
/// Generate ClusterCreateParameters object for 2.X cluster with only Hadoop.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create2XClusterForMapReduceTemplate(HDInsight.ClusterCreateParametersV2 inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var cluster = new ClusterCreateParameters {
DnsName = inputs.Name, Version = inputs.Version
};
var remoteDesktopSettings = (string.IsNullOrEmpty(inputs.RdpUsername))
? new RemoteDesktopSettings()
{
IsEnabled = false
}
: new RemoteDesktopSettings()
{
IsEnabled = true,
AuthenticationCredential = new UsernamePasswordCredential()
{
Username = inputs.RdpUsername,
Password = inputs.RdpPassword
},
RemoteAccessExpiry = (DateTime)inputs.RdpAccessExpiry
};
var headnodeRole = new ClusterRole
{
FriendlyName = "HeadNodeRole",
InstanceCount = 2,
VMSizeAsString = inputs.HeadNodeSize,
RemoteDesktopSettings = remoteDesktopSettings
};
var workernodeRole = new ClusterRole
{
InstanceCount = inputs.ClusterSizeInNodes,
FriendlyName = "WorkerNodeRole",
VMSizeAsString = inputs.DataNodeSize,
RemoteDesktopSettings = remoteDesktopSettings
};
var zookeeperRole = new ClusterRole
{
InstanceCount = 3,
FriendlyName = "ZKRole",
VMSizeAsString = VmSize.Small.ToString(),
RemoteDesktopSettings = remoteDesktopSettings
};
cluster.ClusterRoleCollection.Add(headnodeRole);
cluster.ClusterRoleCollection.Add(workernodeRole);
cluster.ClusterRoleCollection.Add(zookeeperRole);
var gateway = new GatewayComponent
{
IsEnabled = true,
RestAuthCredential = new UsernamePasswordCredential {
Username = inputs.UserName, Password = inputs.Password
}
};
cluster.Components.Add(gateway);
cluster.Location = inputs.Location;
// Adding MapReduce component
MapReduceComponent mapReduce = new MapReduceComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
ConfigMapReduceComponent(mapReduce, inputs);
cluster.Components.Add(mapReduce);
// Adding Hive component
HiveComponent hive = new HiveComponent {
HeadNodeRole = headnodeRole
};
ConfigHiveComponent(hive, inputs);
cluster.Components.Add(hive);
// Adding config action component if needed
if (inputs.ConfigActions != null && inputs.ConfigActions.Count > 0)
{
CustomActionComponent configAction = new CustomActionComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
AddConfigActionComponent(configAction, inputs, headnodeRole, workernodeRole, zookeeperRole);
cluster.Components.Add(configAction);
}
// Adding Oozie component
OozieComponent oozie = new OozieComponent {
HeadNodeRole = headnodeRole
};
ConfigOozieComponent(oozie, inputs);
cluster.Components.Add(oozie);
// Adding Hdfs component
HdfsComponent hdfs = new HdfsComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
ConfigHdfsComponent(hdfs, inputs);
cluster.Components.Add(hdfs);
// Adding HadoopCore component
HadoopCoreComponent hadoopCore = new HadoopCoreComponent();
ConfigHadoopCoreComponent(hadoopCore, inputs);
cluster.Components.Add(hadoopCore);
ConfigVirtualNetwork(cluster, inputs);
return(cluster);
}
/// <summary>
/// Generate ClusterCreateParameters object for 3.X cluster with only Hadoop.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create3XClusterFromMapReduceTemplate(HDInsight.ClusterCreateParametersV2 inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var cluster = new ClusterCreateParameters
{
DnsName = inputs.Name,
Version = inputs.Version
};
var headnodeRole = new ClusterRole
{
FriendlyName = "HeadNodeRole",
InstanceCount = 2,
VMSizeAsString = inputs.HeadNodeSize,
};
var workernodeRole = new ClusterRole
{
InstanceCount = inputs.ClusterSizeInNodes,
FriendlyName = "WorkerNodeRole",
VMSizeAsString = inputs.DataNodeSize,
};
var zookeeperRole = new ClusterRole
{
InstanceCount = 3,
FriendlyName = "ZKRole",
VMSizeAsString = inputs.ZookeeperNodeSize ?? VmSize.Small.ToString(),
};
cluster.ClusterRoleCollection.Add(headnodeRole);
cluster.ClusterRoleCollection.Add(workernodeRole);
cluster.ClusterRoleCollection.Add(zookeeperRole);
var gateway = new GatewayComponent
{
IsEnabled = true,
RestAuthCredential = new UsernamePasswordCredential {
Username = inputs.UserName, Password = inputs.Password
}
};
cluster.Components.Add(gateway);
cluster.Location = inputs.Location;
//Add yarn component
YarnComponent yarn = new YarnComponent {
ResourceManagerRole = headnodeRole, NodeManagerRole = workernodeRole,
};
ConfigYarnComponent(yarn, inputs);
MapReduceApplication mapreduceApp = new MapReduceApplication();
ConfigMapReduceApplication(mapreduceApp, inputs);
yarn.Applications.Add(mapreduceApp);
cluster.Components.Add(yarn);
// Adding Hive component
HiveComponent hive = new HiveComponent {
HeadNodeRole = headnodeRole
};
ConfigHiveComponent(hive, inputs);
cluster.Components.Add(hive);
// Adding config action component if needed
if (inputs.ConfigActions != null && inputs.ConfigActions.Count > 0)
{
CustomActionComponent configAction = new CustomActionComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
AddConfigActionComponent(configAction, inputs, headnodeRole, workernodeRole);
cluster.Components.Add(configAction);
}
// Adding Oozie component
OozieComponent oozie = new OozieComponent {
HeadNodeRole = headnodeRole
};
ConfigOozieComponent(oozie, inputs);
cluster.Components.Add(oozie);
// Adding Hdfs component
HdfsComponent hdfs = new HdfsComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
ConfigHdfsComponent(hdfs, inputs);
cluster.Components.Add(hdfs);
// Adding HadoopCore component
HadoopCoreComponent hadoopCore = new HadoopCoreComponent();
ConfigHadoopCoreComponent(hadoopCore, inputs);
cluster.Components.Add(hadoopCore);
// Adding Zookeeper component
cluster.Components.Add(new ZookeeperComponent {
ZookeeperRole = zookeeperRole
});
ConfigVirtualNetwork(cluster, inputs);
return(cluster);
}
