public static IEnumerable <WabStorageAccountConfiguration> GetAdditionalStorageAccountFromFromPayloadObject(
Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters cluster)
{
// For Yarn clusters (HDI version starts from 3.0), the additional storage account is for MapReduceApplication
YarnComponent yarn = cluster.Components.OfType <YarnComponent>().Single();
MapReduceApplication mrApp = yarn.Applications.OfType <MapReduceApplication>().Single();
var additionalStorageAccounts = mrApp.AdditionalStorageContainers.ToList();
if (additionalStorageAccounts.Any())
{
var result = (from BlobContainerCredentialBackedResource tem in additionalStorageAccounts
select new WabStorageAccountConfiguration(tem.AccountDnsName, tem.Key, tem.BlobContainerName)).ToList();
return(result);
}
return(Enumerable.Empty <WabStorageAccountConfiguration>());
}
public static WabStorageAccountConfiguration GetDefaultStorageAccountFromFromPayloadObject(
Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters cluster)
{
// For Yarn clusters (version later than 3.0 inclusive), the default storage account is for MapReduceApplication
YarnComponent yarn = cluster.Components.OfType <YarnComponent>().Single();
MapReduceApplication mrApp = yarn.Applications.OfType <MapReduceApplication>().Single();
if (mrApp.DefaultStorageAccountAndContainer.ShouldProvisionNew)
{
return(null);
}
return(new WabStorageAccountConfiguration(
mrApp.DefaultStorageAccountAndContainer.AccountDnsName,
mrApp.DefaultStorageAccountAndContainer.Key,
mrApp.DefaultStorageAccountAndContainer.BlobContainerName));
}
private static void ConfigYarnComponent(YarnComponent yarn, HDInsight.ClusterCreateParametersV2 inputs)
{
yarn.Configuration.AddRange(inputs.YarnConfiguration.Select(prop => new Property {
Name = prop.Key, Value = prop.Value
}));
}
/// <summary>
/// Generate ClusterCreateParameters object for 3.X cluster with only Hadoop.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create3XClusterFromMapReduceTemplate(HDInsight.ClusterCreateParametersV2 inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var remoteDesktopSettings = (string.IsNullOrEmpty(inputs.RdpUsername))
? new RemoteDesktopSettings()
{
IsEnabled = false
}
: new RemoteDesktopSettings()
{
IsEnabled = true,
AuthenticationCredential = new UsernamePasswordCredential()
{
Username = inputs.RdpUsername,
Password = inputs.RdpPassword
},
RemoteAccessExpiry = (DateTime)inputs.RdpAccessExpiry
};
var cluster = new ClusterCreateParameters
{
DnsName = inputs.Name,
Version = inputs.Version,
};
var headnodeRole = new ClusterRole
{
FriendlyName = "HeadNodeRole",
InstanceCount = 2,
VMSizeAsString = inputs.HeadNodeSize,
RemoteDesktopSettings = remoteDesktopSettings
};
var workernodeRole = new ClusterRole
{
InstanceCount = inputs.ClusterSizeInNodes,
FriendlyName = "WorkerNodeRole",
VMSizeAsString = inputs.DataNodeSize,
RemoteDesktopSettings = remoteDesktopSettings
};
var zookeeperRole = new ClusterRole
{
InstanceCount = 3,
FriendlyName = "ZKRole",
VMSizeAsString = inputs.ZookeeperNodeSize ?? VmSize.Small.ToString(),
RemoteDesktopSettings = remoteDesktopSettings
};
cluster.ClusterRoleCollection.Add(headnodeRole);
cluster.ClusterRoleCollection.Add(workernodeRole);
cluster.ClusterRoleCollection.Add(zookeeperRole);
var gateway = new GatewayComponent
{
IsEnabled = true,
RestAuthCredential = new UsernamePasswordCredential {
Username = inputs.UserName, Password = inputs.Password
}
};
cluster.Components.Add(gateway);
cluster.Location = inputs.Location;
//Add yarn component
YarnComponent yarn = new YarnComponent {
ResourceManagerRole = headnodeRole, NodeManagerRole = workernodeRole,
};
ConfigYarnComponent(yarn, inputs);
MapReduceApplication mapreduceApp = new MapReduceApplication();
ConfigMapReduceApplication(mapreduceApp, inputs);
yarn.Applications.Add(mapreduceApp);
cluster.Components.Add(yarn);
// Adding Hive component
HiveComponent hive = new HiveComponent {
HeadNodeRole = headnodeRole
};
ConfigHiveComponent(hive, inputs);
cluster.Components.Add(hive);
// Adding config action component if needed
if (inputs.ConfigActions != null && inputs.ConfigActions.Count > 0)
{
CustomActionComponent configAction = new CustomActionComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
AddConfigActionComponent(configAction, inputs, headnodeRole, workernodeRole, zookeeperRole);
cluster.Components.Add(configAction);
}
// Adding Oozie component
OozieComponent oozie = new OozieComponent {
HeadNodeRole = headnodeRole
};
ConfigOozieComponent(oozie, inputs);
cluster.Components.Add(oozie);
// Adding Hdfs component
HdfsComponent hdfs = new HdfsComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
ConfigHdfsComponent(hdfs, inputs);
cluster.Components.Add(hdfs);
// Adding HadoopCore component
HadoopCoreComponent hadoopCore = new HadoopCoreComponent();
ConfigHadoopCoreComponent(hadoopCore, inputs);
cluster.Components.Add(hadoopCore);
// Adding Zookeeper component
cluster.Components.Add(new ZookeeperComponent {
ZookeeperRole = zookeeperRole
});
ConfigVirtualNetwork(cluster, inputs);
return(cluster);
}
