private static AzureHDInsightClusterConfiguration GetAzureHDInsightClusterConfigurationV3(
ClusterDetails cluster, Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters clusterCreateDetails)
{
var azureClusterConfig = new AzureHDInsightClusterConfiguration();
var yarn = clusterCreateDetails.Components.OfType<YarnComponent>().Single();
var mapreduce = yarn.Applications.OfType<MapReduceApplication>().Single();
var hive = clusterCreateDetails.Components.OfType<HiveComponent>().Single();
var oozie = clusterCreateDetails.Components.OfType<OozieComponent>().Single();
var hdfs = clusterCreateDetails.Components.OfType<HdfsComponent>().Single();
var hadoopCore = clusterCreateDetails.Components.OfType<HadoopCoreComponent>().Single();
if (hadoopCore.CoreSiteXmlProperties.Any())
{
AssertConfigOptionsAllowedMay2014(cluster, hadoopCore.CoreSiteXmlProperties, "Core");
azureClusterConfig.Core.AddRange(
hadoopCore.CoreSiteXmlProperties.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (hive.HiveSiteXmlProperties.Any())
{
AssertConfigOptionsAllowedMay2014(cluster, hive.HiveSiteXmlProperties, "Hive");
azureClusterConfig.Hive.AddRange(hive.HiveSiteXmlProperties.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (hdfs.HdfsSiteXmlProperties.Any())
{
AssertConfigOptionsAllowedMay2014(cluster, hdfs.HdfsSiteXmlProperties, "Hdfs");
azureClusterConfig.Hdfs.AddRange(hdfs.HdfsSiteXmlProperties.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (mapreduce.MapRedSiteXmlProperties.Any())
{
AssertConfigOptionsAllowedMay2014(cluster, mapreduce.MapRedSiteXmlProperties, "MapReduce");
azureClusterConfig.MapReduce.AddRange(
mapreduce.MapRedSiteXmlProperties.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (oozie.Configuration.Any())
{
AssertConfigOptionsAllowedMay2014(cluster, oozie.Configuration, "Oozie");
azureClusterConfig.Oozie.AddRange(oozie.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (yarn.Configuration.Any())
{
AssertConfigOptionsAllowedMay2014(cluster, yarn.Configuration, "Yarn");
azureClusterConfig.Yarn.AddRange(oozie.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
return azureClusterConfig;
}
private static AzureHDInsightClusterConfiguration GetAzureHDInsightClusterConfiguration(
ClusterDetails cluster, ClusterContainer clusterCreateDetails)
{
var azureClusterConfig = new AzureHDInsightClusterConfiguration();
if (clusterCreateDetails.Settings.Core != null && clusterCreateDetails.Settings.Core.Configuration != null)
{
AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.Core.Configuration, "Core");
azureClusterConfig.Core.AddRange(
clusterCreateDetails.Settings.Core.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (clusterCreateDetails.Settings.Hive != null && clusterCreateDetails.Settings.Hive.Configuration != null)
{
AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.Hive.Configuration, "Hive");
azureClusterConfig.Hive.AddRange(
clusterCreateDetails.Settings.Hive.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (clusterCreateDetails.Settings.Hdfs != null && clusterCreateDetails.Settings.Hdfs.Configuration != null)
{
AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.Hdfs.Configuration, "Hdfs");
azureClusterConfig.Hdfs.AddRange(
clusterCreateDetails.Settings.Hdfs.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (clusterCreateDetails.Settings.MapReduce != null && clusterCreateDetails.Settings.MapReduce.Configuration != null)
{
AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.MapReduce.Configuration, "MapReduce");
azureClusterConfig.MapReduce.AddRange(
clusterCreateDetails.Settings.MapReduce.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (clusterCreateDetails.Settings.Oozie != null && clusterCreateDetails.Settings.Oozie.Configuration != null)
{
AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.Oozie.Configuration, "Oozie");
azureClusterConfig.Oozie.AddRange(
clusterCreateDetails.Settings.Oozie.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
return azureClusterConfig;
}
private static AzureHDInsightClusterConfiguration GetWellknownClusterConfiguration()
{
var config = new AzureHDInsightClusterConfiguration();
config.Core.AddRange(
new List<KeyValuePair<string, string>>()
{
new KeyValuePair<string, string>("fs.default.name", Hadoop.Client.Constants.WabsProtocolSchemeName + "*****@*****.**"),
new KeyValuePair<string, string>("hadoop.tmp.dir", "/hdfs/tmp"),
new KeyValuePair<string, string>("fs.trash.interval", "60"),
new KeyValuePair<string, string>("fs.checkpoint.dir", "c:\\hdfs\\2nn"),
new KeyValuePair<string, string>("fs.checkpoint.edits.dir", "c:\\hdfs\\2nn"),
new KeyValuePair<string, string>("fs.checkpoint.period", "1800"),
new KeyValuePair<string, string>("fs.checkpoint.size", "67108864"),
new KeyValuePair<string, string>("fs.azure.selfthrottling.write.factor", "1.000000"),
new KeyValuePair<string, string>("fs.azure.buffer.dir", "/tmp"),
new KeyValuePair<string, string>("topology.script.file.name", "E:\\approot\\bin\\AzureTopology.exe"),
new KeyValuePair<string, string>("io.file.buffer.size", "131072"),
new KeyValuePair<string, string>("hadoop.proxyuser.hdp.groups", "oozieusers"),
new KeyValuePair<string, string>("dfs.namenode.rpc-address", "hdfs://*****:*****@e87d4x221e")
});
config.MapReduce.AddRange(
(new List<KeyValuePair<string, string>>()
{
new KeyValuePair<string, string>("mapred.system.dir", "hdfs://*****:*****@e87d4x221e"),
new KeyValuePair<string, string>(
"oozie.service.JPAService.jdbc.password",
"EX5M8nGYh9Vho8dL3ul2UMhcyFH8kcB9o6YJ6CsqvoBujch5yGF1qCeNvYYXEgn4RK5aCfrKwPNtR6bBAWHCowgahkU4wYfV2pKhrtJVQpOCeZ"),
new KeyValuePair<string, string>("oozie.service.JPAService.pool.max.active.conn", "10"),
new KeyValuePair<string, string>("oozie.service.HadoopAccessorService.kerberos.enabled", "false"),
new KeyValuePair<string, string>("local.realm", "LOCALHOST"),
new KeyValuePair<string, string>("oozie.service.HadoopAccessorService.keytab.file", "${user.home}/oozie.keytab"),
new KeyValuePair<string, string>(
"oozie.service.HadoopAccessorService.kerberos.principal", "${user.name}/localhost@${local.realm}"),
new KeyValuePair<string, string>("oozie.service.HadoopAccessorService.jobTracker.whitelist", ""),
new KeyValuePair<string, string>("oozie.service.HadoopAccessorService.nameNode.whitelist", ""),
new KeyValuePair<string, string>("oozie.service.HadoopAccessorService.hadoop.configurations", "*=hadoop-conf"),
new KeyValuePair<string, string>("oozie.service.WorkflowAppService.system.libpath", "/user/${user.name}/share/lib"),
new KeyValuePair<string, string>("use.system.libpath.for.mapreduce.and.pig.jobs", "false"),
new KeyValuePair<string, string>("oozie.authentication.changeType", "simple"),
new KeyValuePair<string, string>("oozie.authentication.token.validity", "36000"),
new KeyValuePair<string, string>("oozie.authentication.signature.secret", "oozie"),
new KeyValuePair<string, string>("oozie.authentication.cookie.domain", ""),
new KeyValuePair<string, string>("oozie.authentication.simple.anonymous.allowed", "true"),
new KeyValuePair<string, string>("oozie.authentication.kerberos.principal", "HTTP/localhost@${local.realm}"),
new KeyValuePair<string, string>("oozie.authentication.kerberos.keytab", "${oozie.service.HadoopAccessorService.keytab.file}"),
new KeyValuePair<string, string>("oozie.authentication.kerberos.name.rules", "DEFAULT"),
new KeyValuePair<string, string>("oozie.service.JPAService.validate.db.connection.eviction.interval", "45000"),
new KeyValuePair<string, string>("oozie.service.JPAService.validate.db.connection", "true"),
new KeyValuePair<string, string>("oozie.service.JPAService.validate.db.connection.eviction.num", "10")
});
return config;
}
