/// <summary>
/// Generate ClusterCreateParameters object for 2.X cluster with only Hadoop.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create2XClusterForMapReduceTemplate(HDInsight.ClusterCreateParameters inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var cluster = new ClusterCreateParameters { DnsName = inputs.Name, Version = inputs.Version };
var headnodeRole = new ClusterRole
{
FriendlyName = "HeadNodeRole",
InstanceCount = 2,
VMSize = inputs.HeadNodeSize.ToVmSize(),
};
var workernodeRole = new ClusterRole
{
InstanceCount = inputs.ClusterSizeInNodes,
FriendlyName = "WorkerNodeRole",
VMSize = VmSize.Large
};
var zookeeperRole = new ClusterRole
{
InstanceCount = 3,
FriendlyName = "ZKRole",
VMSize = VmSize.Small
};
cluster.ClusterRoleCollection.Add(headnodeRole);
cluster.ClusterRoleCollection.Add(workernodeRole);
cluster.ClusterRoleCollection.Add(zookeeperRole);
var gateway = new GatewayComponent
{
IsEnabled = true,
RestAuthCredential = new UsernamePasswordCredential { Username = inputs.UserName, Password = inputs.Password }
};
cluster.Components.Add(gateway);
cluster.Location = inputs.Location;
// Adding MapReduce component
MapReduceComponent mapReduce = new MapReduceComponent { HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole };
ConfigMapReduceComponent(mapReduce, inputs);
cluster.Components.Add(mapReduce);
// Adding Hive component
HiveComponent hive = new HiveComponent { HeadNodeRole = headnodeRole };
ConfigHiveComponent(hive, inputs);
cluster.Components.Add(hive);
// Adding config action component if needed
if (inputs.ConfigActions != null && inputs.ConfigActions.Count > 0)
{
CustomActionComponent configAction = new CustomActionComponent { HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole };
AddConfigActionComponent(configAction, inputs, headnodeRole, workernodeRole);
cluster.Components.Add(configAction);
}
// Adding Oozie component
OozieComponent oozie = new OozieComponent { HeadNodeRole = headnodeRole };
ConfigOozieComponent(oozie, inputs);
cluster.Components.Add(oozie);
// Adding Hdfs component
HdfsComponent hdfs = new HdfsComponent { HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole };
ConfigHdfsComponent(hdfs, inputs);
cluster.Components.Add(hdfs);
// Adding HadoopCore component
HadoopCoreComponent hadoopCore = new HadoopCoreComponent();
ConfigHadoopCoreComponent(hadoopCore, inputs);
cluster.Components.Add(hadoopCore);
ConfigVirtualNetwork(cluster, inputs);
return cluster;
}
private static void AddConfigActionComponent(CustomActionComponent configAction, HDInsight.ClusterCreateParametersV2 inputs, ClusterRole headnodeRole, ClusterRole workernodeRole, ClusterRole zookeperRole)
{
configAction.CustomActions = new CustomActionList();
// Converts config action from PS/SDK to wire contract.
foreach (ConfigAction ca in inputs.ConfigActions)
{
CustomAction newConfigAction;
// Based on the config action type defined in SDK, convert them to config action defined in wire contract.
ScriptAction sca = ca as ScriptAction;
if (sca != null)
{
newConfigAction = new ScriptCustomAction
{
Name = ca.Name,
Uri = sca.Uri,
Parameters = sca.Parameters
};
}
else
{
throw new NotSupportedException("No such config action supported.");
}
newConfigAction.ClusterRoleCollection = new ClusterRoleCollection();
// Add in cluster role collection for each config action.
foreach (ClusterNodeType clusterRoleType in ca.ClusterRoleCollection)
{
if (clusterRoleType == ClusterNodeType.HeadNode)
{
newConfigAction.ClusterRoleCollection.Add(headnodeRole);
}
else if (clusterRoleType == ClusterNodeType.DataNode)
{
newConfigAction.ClusterRoleCollection.Add(workernodeRole);
}
else if (clusterRoleType == ClusterNodeType.ZookeperNode)
{
if (inputs.ClusterType.Equals(ClusterType.HBase) || inputs.ClusterType.Equals(ClusterType.Storm))
{
newConfigAction.ClusterRoleCollection.Add(zookeperRole);
}
else
{
throw new NotSupportedException(string.Format("Customization of zookeper nodes only supported for cluster types {0} and {1}",
ClusterType.HBase.ToString(), ClusterType.Storm.ToString()));
}
}
else
{
throw new NotSupportedException("No such node type supported.");
}
}
configAction.CustomActions.Add(newConfigAction);
}
}
private static void AddConfigActionComponent(CustomActionComponent configAction, HDInsight.ClusterCreateParameters inputs, ClusterRole headnodeRole, ClusterRole workernodeRole)
{
configAction.CustomActions = new CustomActionList();
// Converts config action from PS/SDK to wire contract.
foreach (ConfigAction ca in inputs.ConfigActions)
{
CustomAction newConfigAction;
// Based on the config action type defined in SDK, convert them to config action defined in wire contract.
ScriptAction sca = ca as ScriptAction;
if (sca != null)
{
newConfigAction = new ScriptCustomAction
{
Name = ca.Name,
Uri = sca.Uri,
Parameters = sca.Parameters
};
}
else
{
throw new NotSupportedException("No such config action supported.");
}
newConfigAction.ClusterRoleCollection = new ClusterRoleCollection();
// Add in cluster role collection for each config action.
foreach (ClusterNodeType clusterRoleType in ca.ClusterRoleCollection)
{
if (clusterRoleType == ClusterNodeType.HeadNode)
{
newConfigAction.ClusterRoleCollection.Add(headnodeRole);
}
else if (clusterRoleType == ClusterNodeType.DataNode)
{
newConfigAction.ClusterRoleCollection.Add(workernodeRole);
}
else
{
throw new NotSupportedException("No such node type supported.");
}
}
configAction.CustomActions.Add(newConfigAction);
}
}
/// <summary>
/// Generate ClusterCreateParameters object for 3.X cluster with only Hadoop.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create3XClusterFromMapReduceTemplate(HDInsight.ClusterCreateParametersV2 inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var remoteDesktopSettings = (string.IsNullOrEmpty(inputs.RdpUsername))
? new RemoteDesktopSettings()
{
IsEnabled = false
}
: new RemoteDesktopSettings()
{
IsEnabled = true,
AuthenticationCredential = new UsernamePasswordCredential()
{
Username = inputs.RdpUsername,
Password = inputs.RdpPassword
},
RemoteAccessExpiry = (DateTime)inputs.RdpAccessExpiry
};
var cluster = new ClusterCreateParameters
{
DnsName = inputs.Name,
Version = inputs.Version,
};
var headnodeRole = new ClusterRole
{
FriendlyName = "HeadNodeRole",
InstanceCount = 2,
VMSizeAsString = inputs.HeadNodeSize,
RemoteDesktopSettings = remoteDesktopSettings
};
var workernodeRole = new ClusterRole
{
InstanceCount = inputs.ClusterSizeInNodes,
FriendlyName = "WorkerNodeRole",
VMSizeAsString = inputs.DataNodeSize,
RemoteDesktopSettings = remoteDesktopSettings
};
var zookeeperRole = new ClusterRole
{
InstanceCount = 3,
FriendlyName = "ZKRole",
VMSizeAsString = inputs.ZookeeperNodeSize ?? VmSize.Small.ToString(),
RemoteDesktopSettings = remoteDesktopSettings
};
cluster.ClusterRoleCollection.Add(headnodeRole);
cluster.ClusterRoleCollection.Add(workernodeRole);
cluster.ClusterRoleCollection.Add(zookeeperRole);
var gateway = new GatewayComponent
{
IsEnabled = true,
RestAuthCredential = new UsernamePasswordCredential { Username = inputs.UserName, Password = inputs.Password }
};
cluster.Components.Add(gateway);
cluster.Location = inputs.Location;
//Add yarn component
YarnComponent yarn = new YarnComponent { ResourceManagerRole = headnodeRole, NodeManagerRole = workernodeRole, };
ConfigYarnComponent(yarn, inputs);
MapReduceApplication mapreduceApp = new MapReduceApplication();
ConfigMapReduceApplication(mapreduceApp, inputs);
yarn.Applications.Add(mapreduceApp);
cluster.Components.Add(yarn);
// Adding Hive component
HiveComponent hive = new HiveComponent { HeadNodeRole = headnodeRole };
ConfigHiveComponent(hive, inputs);
cluster.Components.Add(hive);
// Adding config action component if needed
if (inputs.ConfigActions != null && inputs.ConfigActions.Count > 0)
{
CustomActionComponent configAction = new CustomActionComponent { HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole };
AddConfigActionComponent(configAction, inputs, headnodeRole, workernodeRole, zookeeperRole);
cluster.Components.Add(configAction);
}
// Adding Oozie component
OozieComponent oozie = new OozieComponent { HeadNodeRole = headnodeRole };
ConfigOozieComponent(oozie, inputs);
cluster.Components.Add(oozie);
// Adding Hdfs component
HdfsComponent hdfs = new HdfsComponent { HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole };
ConfigHdfsComponent(hdfs, inputs);
cluster.Components.Add(hdfs);
// Adding HadoopCore component
HadoopCoreComponent hadoopCore = new HadoopCoreComponent();
ConfigHadoopCoreComponent(hadoopCore, inputs);
cluster.Components.Add(hadoopCore);
// Adding Zookeeper component
cluster.Components.Add(new ZookeeperComponent { ZookeeperRole = zookeeperRole });
ConfigVirtualNetwork(cluster, inputs);
return cluster;
}