private static void ConfigMapReduceApplication(MapReduceApplication mapReduceApp, HDInsight.ClusterCreateParameters inputs)
{
mapReduceApp.MapRedSiteXmlProperties.AddRange(
inputs.MapReduceConfiguration.ConfigurationCollection.Select(prop => new Property { Name = prop.Key, Value = prop.Value }));
mapReduceApp.CapacitySchedulerConfiguration.AddRange(
inputs.MapReduceConfiguration.CapacitySchedulerConfigurationCollection.Select(
prop => new Property { Name = prop.Key, Value = prop.Value }));
mapReduceApp.DefaultStorageAccountAndContainer = new BlobContainerCredentialBackedResource()
{
AccountDnsName = inputs.DefaultStorageAccountName,
BlobContainerName = inputs.DefaultStorageContainer,
Key = inputs.DefaultStorageAccountKey
};
if (inputs.AdditionalStorageAccounts.Any())
{
mapReduceApp.AdditionalStorageContainers.AddRange(
inputs.AdditionalStorageAccounts.Select(
storageAccount =>
new BlobContainerCredentialBackedResource()
{
AccountDnsName = storageAccount.Name,
BlobContainerName = storageAccount.Container,
Key = storageAccount.Key
}));
}
}
/// <summary>
/// Generate ClusterCreateParameters object for 3.X cluster with only Hadoop.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create3XClusterFromMapReduceTemplate(HDInsight.ClusterCreateParameters inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var cluster = new ClusterCreateParameters
{
DnsName = inputs.Name,
Version = inputs.Version
};
var headnodeRole = new ClusterRole
{
FriendlyName = "HeadNodeRole",
InstanceCount = 2,
VMSize = inputs.HeadNodeSize.ToVmSize()
};
var workernodeRole = new ClusterRole
{
InstanceCount = inputs.ClusterSizeInNodes,
FriendlyName = "WorkerNodeRole",
VMSize = VmSize.Large
};
var zookeeperRole = new ClusterRole
{
InstanceCount = 3,
FriendlyName = "ZKRole",
VMSize = VmSize.Small
};
cluster.ClusterRoleCollection.Add(headnodeRole);
cluster.ClusterRoleCollection.Add(workernodeRole);
cluster.ClusterRoleCollection.Add(zookeeperRole);
var gateway = new GatewayComponent
{
IsEnabled = true,
RestAuthCredential = new UsernamePasswordCredential { Username = inputs.UserName, Password = inputs.Password }
};
cluster.Components.Add(gateway);
cluster.Location = inputs.Location;
//Add yarn component
YarnComponent yarn = new YarnComponent { ResourceManagerRole = headnodeRole, NodeManagerRole = workernodeRole, };
ConfigYarnComponent(yarn, inputs);
MapReduceApplication mapreduceApp = new MapReduceApplication();
ConfigMapReduceApplication(mapreduceApp, inputs);
yarn.Applications.Add(mapreduceApp);
cluster.Components.Add(yarn);
// Adding Hive component
HiveComponent hive = new HiveComponent { HeadNodeRole = headnodeRole };
ConfigHiveComponent(hive, inputs);
cluster.Components.Add(hive);
// Adding config action component if needed
if (inputs.ConfigActions != null && inputs.ConfigActions.Count > 0)
{
CustomActionComponent configAction = new CustomActionComponent { HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole };
AddConfigActionComponent(configAction, inputs, headnodeRole, workernodeRole);
cluster.Components.Add(configAction);
}
// Adding Oozie component
OozieComponent oozie = new OozieComponent { HeadNodeRole = headnodeRole };
ConfigOozieComponent(oozie, inputs);
cluster.Components.Add(oozie);
// Adding Hdfs component
HdfsComponent hdfs = new HdfsComponent { HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole };
ConfigHdfsComponent(hdfs, inputs);
cluster.Components.Add(hdfs);
// Adding HadoopCore component
HadoopCoreComponent hadoopCore = new HadoopCoreComponent();
ConfigHadoopCoreComponent(hadoopCore, inputs);
cluster.Components.Add(hadoopCore);
// Adding Zookeeper component
cluster.Components.Add(new ZookeeperComponent { ZookeeperRole = zookeeperRole });
ConfigVirtualNetwork(cluster, inputs);
return cluster;
}
