public ClusterCreateParameters(ClusterCreateParameters parameters)
{
this.Location = parameters.Location;
this.UserName = parameters.UserName;
this.Password = parameters.Password;
this.RdpUsername = parameters.RdpUsername;
this.RdpPassword = parameters.RdpPassword;
this.RdpAccessExpiry = parameters.RdpAccessExpiry;
this.ClusterSizeInNodes = parameters.ClusterSizeInNodes;
this.Version = parameters.Version;
this.HeadNodeSize = parameters.HeadNodeSize;
this.WorkerNodeSize = parameters.WorkerNodeSize;
this.ZookeeperNodeSize = parameters.ZookeeperNodeSize;
this.EdgeNodeSize = parameters.EdgeNodeSize;
this.AdditionalStorageAccounts = parameters.AdditionalStorageAccounts;
this.ScriptActions = parameters.ScriptActions;
this.OozieMetastore = parameters.OozieMetastore;
this.HiveMetastore = parameters.HiveMetastore;
this.Configurations = parameters.Configurations;
this.ClusterType = parameters.ClusterType;
this.VirtualNetworkId = parameters.VirtualNetworkId;
this.SubnetName = parameters.SubnetName;
this.ClusterTier = parameters.ClusterTier;
this.SshUserName = parameters.SshUserName;
this.SshPassword = parameters.SshPassword;
this.SshPublicKey = parameters.SshPublicKey;
this.Principal = parameters.Principal;
this.SecurityProfile = parameters.SecurityProfile;
this.WorkerNodeDataDisksGroups = parameters.WorkerNodeDataDisksGroups;
this.ComponentVersion = parameters.ComponentVersion;
this.DefaultStorageInfo = parameters.DefaultStorageInfo;
this.Tags = parameters.Tags;
this.ClusterIdentity = parameters.ClusterIdentity;
this.DiskEncryptionProperties = parameters.DiskEncryptionProperties;
}
public void AssertDefaultVmTypes(string clusterType, string expectedheadNodeVmType, string expectedWorkerNodeVmType)
{
var createParams = new ClusterCreateParameters()
{
ClusterType = clusterType,
};
Assert.Equal(expectedheadNodeVmType, ClusterOperations.GetHeadNodeSize(createParams));
Assert.Equal(expectedWorkerNodeVmType, ClusterOperations.GetWorkerNodeSize(createParams));
}
/// <summary>
/// Begins creating a new HDInsight cluster with the specified
/// parameters.
/// </summary>
/// <param name='resourceGroupName'>
/// Required. The name of the resource group.
/// </param>
/// <param name='clusterName'>
/// Required. The name of the cluster.
/// </param>
/// <param name='clusterCreateParameters'>
/// Required. The cluster create request.
/// </param>
/// <param name='cancellationToken'>
/// Cancellation token.
/// </param>
/// <returns>
/// The CreateCluster operation response.
/// </returns>
public async Task<ClusterCreateResponse> BeginCreatingAsync(string resourceGroupName, string clusterName, ClusterCreateParameters clusterCreateParameters, CancellationToken cancellationToken)
{
try
{
var createParamsExtended = GetExtendedClusterCreateParameters(clusterName, clusterCreateParameters);
return await BeginCreatingAsync(resourceGroupName, clusterName, createParamsExtended, cancellationToken);
}
catch (Exception ex)
{
throw new CloudException(ex.Message);
}
}
/// <summary>
/// Creates a new HDInsight cluster with the specified parameters.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.IClusterOperations.
/// </param>
/// <param name='resourceGroupName'>
/// Required. The name of the resource group.
/// </param>
/// <param name='clusterName'>
/// Required. The name of the cluster.
/// </param>
/// <param name='clusterCreateParameters'>
/// Required. The cluster create request.
/// </param>
/// <returns>
/// The GetCluster operation response.
/// </returns>
public static ClusterGetResponse Create(this IClusterOperations operations, string resourceGroupName,
string clusterName, ClusterCreateParameters clusterCreateParameters)
{
return
Task.Factory.StartNew(
(object s) =>
((IClusterOperations) s).CreateAsync(resourceGroupName, clusterName, clusterCreateParameters),
operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default)
.Unwrap()
.GetAwaiter()
.GetResult();
}
public void TestCreateDefaultFsAzureBlobClusterContainerNotSpecified()
{
string clusterName = "cluster";
var clusterCreateParams = new ClusterCreateParameters
{
ClusterSizeInNodes = 3,
ClusterType = "Hadoop",
WorkerNodeSize = "Large",
DefaultStorageInfo = new AzureStorageInfo("storageAccountName", "storageAccountKey"),
OSType = OSType.Linux,
UserName = "******",
Password = "******",
Location = "East US",
SshUserName = "******",
SshPassword = "******",
Version = "3.2"
};
ClusterOperations op = new ClusterOperations(new HDInsightManagementClient());
var extendedParams = op.GetExtendedClusterCreateParameters(clusterName, clusterCreateParams);
AssertContainerMatchesNameProvided(extendedParams, clusterName);
}
private static Dictionary<string, Dictionary<string, string>> GetConfigurations(string clusterName,
ClusterCreateParameters clusterCreateParameters)
{
var configurations = clusterCreateParameters.Configurations;
//Core Config
var coreConfigExists = true;
Dictionary<string, string> coreConfig;
configurations.TryGetValue(ConfigurationKey.CoreSite, out coreConfig);
if (coreConfig == null)
{
coreConfigExists = false;
coreConfig = new Dictionary<string, string>();
}
if (!coreConfig.ContainsKey("fs.defaultFS"))
{
var storageAccountNameKey = "fs.defaultFS";
if (clusterCreateParameters.Version != null && clusterCreateParameters.Version.Equals("2.1"))
{
storageAccountNameKey = "fs.default.name";
}
var container = !string.IsNullOrEmpty(clusterCreateParameters.DefaultStorageContainer)
? clusterCreateParameters.DefaultStorageContainer
: clusterName;
coreConfig.Add(storageAccountNameKey,
string.Format("wasb://{0}@{1}", container, clusterCreateParameters.DefaultStorageAccountName));
}
var defaultStorageConfigKey = string.Format("fs.azure.account.key.{0}",
clusterCreateParameters.DefaultStorageAccountName);
if (!coreConfig.ContainsKey(defaultStorageConfigKey))
{
coreConfig.Add(defaultStorageConfigKey, clusterCreateParameters.DefaultStorageAccountKey);
}
foreach (var storageAccount in clusterCreateParameters.AdditionalStorageAccounts)
{
var configKey = string.Format("fs.azure.account.key.{0}", storageAccount.Key);
if (!coreConfig.ContainsKey(configKey))
{
coreConfig.Add(configKey, storageAccount.Value);
}
}
if (!coreConfigExists)
{
configurations.Add(ConfigurationKey.CoreSite, coreConfig);
}
else
{
configurations[ConfigurationKey.CoreSite] = coreConfig;
}
//Gateway Config
Dictionary<string, string> gatewayConfig;
configurations.TryGetValue(ConfigurationKey.Gateway, out gatewayConfig);
if (gatewayConfig != null)
{
return configurations;
}
gatewayConfig = new Dictionary<string, string>();
if (!string.IsNullOrEmpty(clusterCreateParameters.UserName))
{
gatewayConfig.Add("restAuthCredential.isEnabled", "true");
gatewayConfig.Add("restAuthCredential.username", clusterCreateParameters.UserName);
gatewayConfig.Add("restAuthCredential.password", clusterCreateParameters.Password);
}
else
{
gatewayConfig.Add("restAuthCredential.isEnabled", "false");
}
configurations.Add(ConfigurationKey.Gateway, gatewayConfig);
return configurations;
}
internal static string GetWorkerNodeSize(ClusterCreateParameters clusterCreateParameters)
{
string workerNodeSize;
if (clusterCreateParameters.WorkerNodeSize != null)
{
workerNodeSize = clusterCreateParameters.WorkerNodeSize;
}
else
{
if (!WorkerNodeDefaultSizes.TryGetValue(clusterCreateParameters.ClusterType, out workerNodeSize))
{
workerNodeSize = "Standard_D3";
}
}
return workerNodeSize;
}
private static string GetHeadNodeSize(ClusterCreateParameters clusterCreateParameters)
{
string headNodeSize;
if (clusterCreateParameters.HeadNodeSize != null)
{
headNodeSize = clusterCreateParameters.HeadNodeSize;
}
else
{
if (clusterCreateParameters.ClusterType.Equals("Hadoop", StringComparison.OrdinalIgnoreCase))
{
headNodeSize = "Standard_D3";
}
else if (clusterCreateParameters.ClusterType.Equals("Spark", StringComparison.OrdinalIgnoreCase))
{
headNodeSize = "Standard_D12";
}
else
{
headNodeSize = "Large";
}
}
return headNodeSize;
}
private static string GetWorkerNodeSize(ClusterCreateParameters clusterCreateParameters)
{
string workerNodeSize;
if (clusterCreateParameters.WorkerNodeSize != null)
{
workerNodeSize = clusterCreateParameters.WorkerNodeSize;
}
else
{
workerNodeSize = clusterCreateParameters.ClusterType == HDInsightClusterType.Spark
? "Standard_D12"
: "Standard_D3";
}
return workerNodeSize;
}
private static IEnumerable<Role> GetRoleCollection(ClusterCreateParameters clusterCreateParameters)
{
//OS Profile
var osProfile = new OsProfile();
if (clusterCreateParameters.OSType == OSType.Windows)
{
RdpSettings rdpSettings = null;
if (!string.IsNullOrEmpty(clusterCreateParameters.RdpUsername))
{
rdpSettings = new RdpSettings
{
UserName = clusterCreateParameters.RdpUsername,
Password = clusterCreateParameters.RdpPassword,
ExpiryDate = clusterCreateParameters.RdpAccessExpiry
};
}
osProfile = new OsProfile
{
WindowsOperatingSystemProfile = new WindowsOperatingSystemProfile
{
RdpSettings = rdpSettings
}
};
}
else if (clusterCreateParameters.OSType == OSType.Linux)
{
var sshPublicKeys = new List<SshPublicKey>();
if (!string.IsNullOrEmpty(clusterCreateParameters.SshPublicKey))
{
var sshPublicKey = new SshPublicKey
{
CertificateData = clusterCreateParameters.SshPublicKey
};
sshPublicKeys.Add(sshPublicKey);
}
SshProfile sshProfile;
if (sshPublicKeys.Count > 0)
{
sshProfile = new SshProfile
{
SshPublicKeys = sshPublicKeys
};
}
else
{
sshProfile = null;
}
osProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
UserName = clusterCreateParameters.SshUserName,
Password = clusterCreateParameters.SshPassword,
SshProfile = sshProfile
}
};
}
//VNet Profile
var vnetProfile = new VirtualNetworkProfile();
if (!string.IsNullOrEmpty(clusterCreateParameters.VirtualNetworkId))
{
vnetProfile.Id = clusterCreateParameters.VirtualNetworkId;
}
if (!string.IsNullOrEmpty(clusterCreateParameters.SubnetName))
{
vnetProfile.SubnetName = clusterCreateParameters.SubnetName;
}
if (string.IsNullOrEmpty(vnetProfile.Id) && string.IsNullOrEmpty(vnetProfile.SubnetName))
{
vnetProfile = null;
}
List<ScriptAction> workernodeactions = null;
List<ScriptAction> headnodeactions = null;
List<ScriptAction> zookeepernodeactions = null;
//Script Actions
foreach (var scriptAction in clusterCreateParameters.ScriptActions)
{
if (scriptAction.Key.ToString().ToLower().Equals("workernode"))
{
workernodeactions = scriptAction.Value;
}
else if (scriptAction.Key.ToString().ToLower().Equals("headnode"))
{
headnodeactions = scriptAction.Value;
}
else if (scriptAction.Key.ToString().ToLower().Equals("zookeepernode"))
{
zookeepernodeactions = scriptAction.Value;
}
}
//Roles
var roles = new List<Role>();
var headNodeSize = GetHeadNodeSize(clusterCreateParameters);
var headNode = new Role
{
Name = "headnode",
TargetInstanceCount = 2,
HardwareProfile = new HardwareProfile
{
VmSize = headNodeSize
},
OsProfile = osProfile,
VirtualNetworkProfile = vnetProfile,
ScriptActions = headnodeactions
};
roles.Add(headNode);
var workerNodeSize = GetWorkerNodeSize(clusterCreateParameters);
var workerNode = new Role
{
Name = "workernode",
TargetInstanceCount = clusterCreateParameters.ClusterSizeInNodes,
HardwareProfile = new HardwareProfile
{
VmSize = workerNodeSize
},
OsProfile = osProfile,
ScriptActions = workernodeactions
};
roles.Add(workerNode);
if (clusterCreateParameters.OSType == OSType.Windows)
{
if (clusterCreateParameters.ClusterType == HDInsightClusterType.Hadoop ||
clusterCreateParameters.ClusterType == HDInsightClusterType.Spark)
{
return roles;
}
}
if (clusterCreateParameters.OSType == OSType.Linux)
{
if (clusterCreateParameters.ClusterType == HDInsightClusterType.Hadoop ||
clusterCreateParameters.ClusterType == HDInsightClusterType.Spark)
{
clusterCreateParameters.ZookeeperNodeSize = "Small";
}
}
string zookeeperNodeSize = clusterCreateParameters.ZookeeperNodeSize ?? "Medium";
var zookeepernode = new Role
{
Name = "zookeepernode",
ScriptActions = zookeepernodeactions,
TargetInstanceCount = 3,
OsProfile = osProfile,
HardwareProfile = new HardwareProfile
{
VmSize = zookeeperNodeSize
}
};
roles.Add(zookeepernode);
return roles;
}
public static ClusterCreateParameters GetCustomCreateParametersIaas()
{
var clusterparams = new ClusterCreateParameters
{
ClusterSizeInNodes = 3,
ClusterType = HDInsightClusterType.Hadoop,
WorkerNodeSize = "Large",
DefaultStorageAccountName = StorageAccountName,
DefaultStorageAccountKey = StorageAccountKey,
OSType = OSType.Linux,
UserName = HttpUser,
Password = HttpPassword,
DefaultStorageContainer = DefaultContainer,
Location = "West US",
SshUserName = SshUser,
SshPassword = SshPassword,
Version = "3.2"
};
return clusterparams;
}
public static ClusterCreateParameters GetCustomVmSizesCreateParametersIaas()
{
var clusterparams = new ClusterCreateParameters
{
ClusterSizeInNodes = 1,
ClusterType = HDInsightClusterType.HBase,
WorkerNodeSize = "Large",
DefaultStorageAccountName = StorageAccountName,
DefaultStorageAccountKey = StorageAccountKey,
OSType = OSType.Linux,
UserName = HttpUser,
Password = HttpPassword,
DefaultStorageContainer = DefaultContainer,
Location = "East US",
SshUserName = SshUser,
SshPassword = SshPassword,
Version = "3.2",
HeadNodeSize = "ExtraLarge",
ZookeeperNodeSize = "Large",
};
return clusterparams;
}
/// <summary>
/// Creates a new HDInsight cluster with the specified parameters.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.IClusterOperations.
/// </param>
/// <param name='resourceGroupName'>
/// Required. The name of the resource group.
/// </param>
/// <param name='clusterName'>
/// Required. The name of the cluster.
/// </param>
/// <param name='clusterCreateParameters'>
/// Required. The cluster create request.
/// </param>
/// <returns>
/// The GetCluster operation response.
/// </returns>
public static Task<ClusterGetResponse> CreateAsync(this IClusterOperations operations, string resourceGroupName,
string clusterName, ClusterCreateParameters clusterCreateParameters)
{
return operations.CreateAsync(resourceGroupName, clusterName, clusterCreateParameters,
CancellationToken.None);
}
public virtual ClusterGetResponse CreateNewCluster(string resourceGroupName, string clusterName, ClusterCreateParameters parameters)
{
return HdInsightManagementClient.Clusters.Create(resourceGroupName, clusterName, parameters);
}
static void Create(string subscriptionId, string resourceGroupName, string clusterDnsName, string clusterLocation,
AzureStorageConfig defaultStorageAccount, int clusterSize, string clusterUsername, string clusterPassword,
string hdInsightVersion,
string clusterType, string osType,
List<AzureStorageConfig> additionalStorageAccounts,
List<SqlAzureConfig> sqlAzureMetaStores,
string virtualNetworkId = null, string subnetName = null,
string headNodeSize = null, string workerNodeSize = null, string zookeeperSize = null)
{
Logger.InfoFormat("ResourceGroup: {0}, Cluster: {1} - Submitting a new cluster deployment request", resourceGroupName, clusterDnsName);
var clusterCreateParameters = new ClusterCreateParameters()
{
ClusterSizeInNodes = clusterSize,
ClusterType = clusterType,
DefaultStorageAccountName = defaultStorageAccount.Name,
DefaultStorageAccountKey = defaultStorageAccount.Key,
DefaultStorageContainer = defaultStorageAccount.Container,
Location = clusterLocation,
Password = clusterPassword,
UserName = clusterUsername,
Version = hdInsightVersion,
OSType = (OSType)Enum.Parse(typeof(OSType), osType.ToString()),
HeadNodeSize = headNodeSize,
WorkerNodeSize = workerNodeSize,
ZookeeperNodeSize = zookeeperSize
};
if (clusterCreateParameters.OSType == OSType.Linux)
{
clusterCreateParameters.SshUserName = config.SshUsername;
if(!String.IsNullOrWhiteSpace(config.SshPassword))
{
clusterCreateParameters.SshPassword = config.SshPassword;
}
else if(File.Exists(config.SshPublicKeyFilePath))
{
var publicKey = File.ReadAllText(config.SshPublicKeyFilePath);
Logger.Debug("SSH RSA Public Key: " + Environment.NewLine + publicKey + Environment.NewLine);
clusterCreateParameters.SshPublicKey = publicKey;
}
else if (String.IsNullOrWhiteSpace(config.SshPassword))
{
clusterCreateParameters.SshPassword = config.ClusterPassword;
}
}
else
{
if (config.AutoEnableRdp)
{
clusterCreateParameters.RdpUsername = config.RdpUsername;
clusterCreateParameters.RdpPassword = config.RdpPassword;
clusterCreateParameters.RdpAccessExpiry = DateTime.Now.AddDays(int.Parse(config.RdpExpirationInDays));
}
}
foreach (var additionalStorageAccount in additionalStorageAccounts)
{
clusterCreateParameters.AdditionalStorageAccounts.Add(additionalStorageAccount.Name, additionalStorageAccount.Key);
}
if (!String.IsNullOrEmpty(virtualNetworkId) && !String.IsNullOrEmpty(subnetName))
{
clusterCreateParameters.VirtualNetworkId = virtualNetworkId;
clusterCreateParameters.SubnetName = subnetName;
}
if (sqlAzureMetaStores != null && sqlAzureMetaStores.Count > 0)
{
var hiveMetastore = sqlAzureMetaStores.FirstOrDefault(s => s.Type.Equals("HiveMetastore"));
if (hiveMetastore != null)
{
clusterCreateParameters.HiveMetastore =
new Metastore(hiveMetastore.Server, hiveMetastore.Database, hiveMetastore.User, hiveMetastore.Password);
}
var oozieMetastore = sqlAzureMetaStores.FirstOrDefault(s => s.Type.Equals("OozieMetastore"));
if (oozieMetastore != null)
{
clusterCreateParameters.OozieMetastore =
new Metastore(oozieMetastore.Server, oozieMetastore.Database, oozieMetastore.User, oozieMetastore.Password);
}
}
var localStopWatch = Stopwatch.StartNew();
var createTask = hdInsightManagementClient.Clusters.CreateAsync(resourceGroupName, clusterDnsName, clusterCreateParameters);
Logger.InfoFormat("Cluster: {0} - Create cluster request submitted with task id: {1}, task status: {2}",
clusterDnsName, createTask.Id, createTask.Status);
Thread.Sleep(pollInterval);
var error = MonitorCreate(resourceGroupName, clusterDnsName, createTask);
if (error)
{
if (config.CleanupOnError)
{
Logger.InfoFormat("{0} - {1}. Submitting a delete request for the failed cluster creation.", Config.ConfigName.CleanupOnError.ToString(), config.CleanupOnError.ToString());
Delete(resourceGroupName, clusterDnsName);
}
else
{
throw new ApplicationException(String.Format("Cluster: {0} - Creation unsuccessful", clusterDnsName));
}
}
else
{
if (config.AutoEnableRdp && clusterCreateParameters.OSType == OSType.Windows)
{
HDInsightManagementCLIHelpers.CreateRdpFile(clusterDnsName, config.RdpUsername, config.RdpPassword);
}
}
}
private static ClusterCreateParameters GetDefaultFsCreateParametersIaas(StorageInfo defaultStorageInfo)
{
var clusterparams = new ClusterCreateParameters
{
ClusterSizeInNodes = 3,
ClusterType = "Hadoop",
WorkerNodeSize = "Large",
DefaultStorageInfo = defaultStorageInfo,
OSType = OSType.Linux,
UserName = HttpUser,
Password = HttpPassword,
Location = "East US",
SshUserName = SshUser,
SshPassword = SshPassword,
Version = "3.2"
};
return clusterparams;
}
private static string GetWorkerNodeSize(ClusterCreateParameters clusterCreateParameters)
{
string workerNodeSize;
if (clusterCreateParameters.WorkerNodeSize != null)
{
workerNodeSize = clusterCreateParameters.WorkerNodeSize;
}
else
{
workerNodeSize = clusterCreateParameters.ClusterType.Equals("Spark", StringComparison.OrdinalIgnoreCase)
? "Standard_D12"
: "Standard_D3";
}
return workerNodeSize;
}
private static string GetHeadNodeSize(ClusterCreateParameters clusterCreateParameters)
{
string headNodeSize;
if (clusterCreateParameters.HeadNodeSize != null)
{
headNodeSize = clusterCreateParameters.HeadNodeSize;
}
else
{
headNodeSize = clusterCreateParameters.ClusterType == HDInsightClusterType.Hadoop ||
clusterCreateParameters.ClusterType == HDInsightClusterType.Spark
? "Standard_D12"
: "Large";
}
return headNodeSize;
}
private static Dictionary<string, Dictionary<string, string>> GetConfigurations(string clusterName,
ClusterCreateParameters clusterCreateParameters)
{
var configurations = clusterCreateParameters.Configurations;
//Core Config
var coreConfigExists = true;
Dictionary<string, string> coreConfig;
configurations.TryGetValue(ConfigurationKey.CoreSite, out coreConfig);
if (coreConfig == null)
{
coreConfigExists = false;
coreConfig = new Dictionary<string, string>();
}
AddDefaultStorageAccountToCoreConfig(clusterName, clusterCreateParameters, coreConfig);
AddWasbStorageAccountKeyToCoreConfig(clusterCreateParameters, coreConfig);
AddAdditionalStorageAcountsToCoreConfig(clusterCreateParameters, coreConfig);
if (!coreConfigExists)
{
configurations.Add(ConfigurationKey.CoreSite, coreConfig);
}
else
{
configurations[ConfigurationKey.CoreSite] = coreConfig;
}
//Gateway Config
Dictionary<string, string> gatewayConfig;
configurations.TryGetValue(ConfigurationKey.Gateway, out gatewayConfig);
if (gatewayConfig == null)
{
gatewayConfig = new Dictionary<string, string>();
}
if (!string.IsNullOrEmpty(clusterCreateParameters.UserName))
{
gatewayConfig.Add("restAuthCredential.isEnabled", "true");
gatewayConfig.Add("restAuthCredential.username", clusterCreateParameters.UserName);
gatewayConfig.Add("restAuthCredential.password", clusterCreateParameters.Password);
}
else
{
gatewayConfig.Add("restAuthCredential.isEnabled", "false");
}
configurations.Add(ConfigurationKey.Gateway, gatewayConfig);
//datalake configs
var datalakeConfigExists = true;
Dictionary<string, string> datalakeConfig;
configurations.TryGetValue(ConfigurationKey.ClusterIdentity, out datalakeConfig);
if (datalakeConfig == null)
{
datalakeConfigExists = false;
}
//Add/override datalake config if principal is provided by user
if (clusterCreateParameters.Principal != null)
{
datalakeConfig = new Dictionary<string, string>();
ServicePrincipal servicePrincipalObj = (ServicePrincipal)clusterCreateParameters.Principal;
datalakeConfig.Add("clusterIdentity.applicationId", servicePrincipalObj.ApplicationId.ToString());
// converting the tenant Id to URI as RP expects this to be URI
datalakeConfig.Add("clusterIdentity.aadTenantId", "https://login.windows.net/" + servicePrincipalObj.AADTenantId.ToString());
datalakeConfig.Add("clusterIdentity.certificate", Convert.ToBase64String(servicePrincipalObj.CertificateFileBytes));
datalakeConfig.Add("clusterIdentity.certificatePassword", servicePrincipalObj.CertificatePassword);
datalakeConfig.Add("clusterIdentity.resourceUri", servicePrincipalObj.ResourceUri.ToString());
if (!datalakeConfigExists)
{
configurations.Add(ConfigurationKey.ClusterIdentity, datalakeConfig);
}
else
{
configurations[ConfigurationKey.ClusterIdentity] = datalakeConfig;
}
}
return configurations;
}
private static void AddAdditionalStorageAcountsToCoreConfig(ClusterCreateParameters clusterCreateParameters, Dictionary<string, string> coreConfig)
{
foreach (var storageAccount in clusterCreateParameters.AdditionalStorageAccounts)
{
var configKey = string.Format(_wasbStorageAccountKeyFormat, storageAccount.Key);
if (!string.IsNullOrEmpty(storageAccount.Value) && !coreConfig.ContainsKey(configKey))
{
coreConfig.Add(configKey, storageAccount.Value);
}
}
}
public static ClusterCreateParameters GetCustomCreateParametersPaas()
{
var clusterparams = new ClusterCreateParameters
{
ClusterSizeInNodes = 3,
ClusterType = HDInsightClusterType.Hadoop,
WorkerNodeSize = "Large",
DefaultStorageAccountName = StorageAccountName,
DefaultStorageAccountKey = StorageAccountKey,
OSType = OSType.Windows,
UserName = HttpUser,
Password = HttpPassword,
DefaultStorageContainer = DefaultContainer,
Location = "West US"
};
var actions = new List<ScriptAction>();
var action = new ScriptAction("action", new Uri("https://uri.com"), "params");
actions.Add(action);
clusterparams.ScriptActions.Add(ClusterNodeType.WorkerNode, actions);
clusterparams.ScriptActions.Add(ClusterNodeType.HeadNode, actions);
return clusterparams;
}
private static void AddWasbStorageAccountKeyToCoreConfig(ClusterCreateParameters clusterCreateParameters, Dictionary<string, string> coreConfig)
{
var storageAccountInfo = clusterCreateParameters.DefaultStorageInfo as AzureStorageInfo;
if (storageAccountInfo != null)
{
string defaultStorageConfigKey = string.Format(_wasbStorageAccountKeyFormat, clusterCreateParameters.DefaultStorageInfo.StorageAccountName);
string storageAccountKey = storageAccountInfo.StorageAccountKey;
if (!string.IsNullOrEmpty(storageAccountKey) && !coreConfig.ContainsKey(defaultStorageConfigKey))
{
coreConfig.Add(defaultStorageConfigKey, storageAccountKey);
}
}
}
private static Dictionary<string, Dictionary<string, string>> GetConfigurations(string clusterName,
ClusterCreateParameters clusterCreateParameters)
{
var configurations = clusterCreateParameters.Configurations;
//Core Config
var coreConfigExists = true;
Dictionary<string, string> coreConfig;
configurations.TryGetValue(ConfigurationKey.CoreSite, out coreConfig);
if (coreConfig == null)
{
coreConfigExists = false;
coreConfig = new Dictionary<string, string>();
}
if (!coreConfig.ContainsKey("fs.defaultFS"))
{
var storageAccountNameKey = "fs.defaultFS";
if (clusterCreateParameters.Version != null && clusterCreateParameters.Version.Equals("2.1"))
{
storageAccountNameKey = "fs.default.name";
}
var container = !string.IsNullOrEmpty(clusterCreateParameters.DefaultStorageContainer)
? clusterCreateParameters.DefaultStorageContainer
: clusterName;
coreConfig.Add(storageAccountNameKey,
string.Format("wasb://{0}@{1}", container, clusterCreateParameters.DefaultStorageAccountName));
}
var defaultStorageConfigKey = string.Format("fs.azure.account.key.{0}",
clusterCreateParameters.DefaultStorageAccountName);
if (!coreConfig.ContainsKey(defaultStorageConfigKey))
{
coreConfig.Add(defaultStorageConfigKey, clusterCreateParameters.DefaultStorageAccountKey);
}
foreach (var storageAccount in clusterCreateParameters.AdditionalStorageAccounts)
{
var configKey = string.Format("fs.azure.account.key.{0}", storageAccount.Key);
if (!coreConfig.ContainsKey(configKey))
{
coreConfig.Add(configKey, storageAccount.Value);
}
}
if (!coreConfigExists)
{
configurations.Add(ConfigurationKey.CoreSite, coreConfig);
}
else
{
configurations[ConfigurationKey.CoreSite] = coreConfig;
}
//Gateway Config
Dictionary<string, string> gatewayConfig;
configurations.TryGetValue(ConfigurationKey.Gateway, out gatewayConfig);
if (gatewayConfig == null)
{
gatewayConfig = new Dictionary<string, string>();
}
if (!string.IsNullOrEmpty(clusterCreateParameters.UserName))
{
gatewayConfig.Add("restAuthCredential.isEnabled", "true");
gatewayConfig.Add("restAuthCredential.username", clusterCreateParameters.UserName);
gatewayConfig.Add("restAuthCredential.password", clusterCreateParameters.Password);
}
else
{
gatewayConfig.Add("restAuthCredential.isEnabled", "false");
}
configurations.Add(ConfigurationKey.Gateway, gatewayConfig);
//datalake configs
var datalakeConfigExists = true;
Dictionary<string, string> datalakeConfig;
configurations.TryGetValue(ConfigurationKey.ClusterIdentity, out datalakeConfig);
if (datalakeConfig == null)
{
datalakeConfigExists = false;
}
//Add/override datalake config if principal is provided by user
if (clusterCreateParameters.Principal != null)
{
datalakeConfig = new Dictionary<string, string>();
ServicePrincipal servicePrincipalObj = (ServicePrincipal)clusterCreateParameters.Principal;
datalakeConfig.Add("clusterIdentity.applicationId", servicePrincipalObj.ApplicationId.ToString());
// converting the tenant Id to URI as RP expects this to be URI
datalakeConfig.Add("clusterIdentity.aadTenantId", "https://login.windows.net/" + servicePrincipalObj.AADTenantId.ToString());
datalakeConfig.Add("clusterIdentity.certificate", Convert.ToBase64String(servicePrincipalObj.CertificateFileBytes));
datalakeConfig.Add("clusterIdentity.certificatePassword", servicePrincipalObj.CertificatePassword);
datalakeConfig.Add("clusterIdentity.resourceUri", servicePrincipalObj.ResourceUri.ToString());
if (!datalakeConfigExists)
{
configurations.Add(ConfigurationKey.ClusterIdentity, datalakeConfig);
}
else
{
configurations[ConfigurationKey.ClusterIdentity] = datalakeConfig;
}
}
return configurations;
}
private ClusterCreateParametersExtended GetExtendedClusterCreateParameters(
string clusterName, ClusterCreateParameters clusterCreateParameters)
{
var createParamsExtended = new ClusterCreateParametersExtended
{
Location = clusterCreateParameters.Location,
Properties = new ClusterCreateProperties
{
ClusterDefinition = new ClusterDefinition
{
ClusterType = clusterCreateParameters.ClusterType
},
ClusterVersion = clusterCreateParameters.Version,
OperatingSystemType = clusterCreateParameters.OSType
}
};
var configurations = GetConfigurations(clusterName, clusterCreateParameters);
if (clusterCreateParameters.HiveMetastore != null)
{
var metastoreConfig = GetMetastoreConfig(clusterCreateParameters.HiveMetastore, clusterCreateParameters.OSType, "Hive");
foreach (var configSet in metastoreConfig)
{
if (configurations.ContainsKey(configSet.Key))
{
foreach (var config in configSet.Value)
{
configurations[configSet.Key].Add(config.Key, config.Value);
}
}
else
{
configurations.Add(configSet.Key, configSet.Value);
}
}
}
if (clusterCreateParameters.OozieMetastore != null)
{
var metastoreConfig = GetMetastoreConfig(clusterCreateParameters.OozieMetastore, clusterCreateParameters.OSType, "oozie");
foreach (var configSet in metastoreConfig)
{
if (configurations.ContainsKey(configSet.Key))
{
foreach (var config in configSet.Value)
{
configurations[configSet.Key].Add(config.Key, config.Value);
}
}
else
{
configurations.Add(configSet.Key, configSet.Value);
}
}
}
var serializedConfig = JsonConvert.SerializeObject(configurations);
createParamsExtended.Properties.ClusterDefinition.Configurations = serializedConfig;
var roles = GetRoleCollection(clusterCreateParameters);
createParamsExtended.Properties.ComputeProfile = new ComputeProfile();
foreach (var role in roles)
{
role.SecurityProfile = clusterCreateParameters.SecurityProfile;
createParamsExtended.Properties.ComputeProfile.Roles.Add(role);
}
return createParamsExtended;
}
private static string GetHeadNodeSize(ClusterCreateParameters clusterCreateParameters)
{
string headNodeSize;
if (clusterCreateParameters.HeadNodeSize != null)
{
headNodeSize = clusterCreateParameters.HeadNodeSize;
}
else
{
switch (clusterCreateParameters.ClusterType)
{
case HDInsightClusterType.Hadoop:
headNodeSize = "Standard_D3";
break;
case HDInsightClusterType.Spark:
headNodeSize = "Standard_D12";
break;
default:
headNodeSize = "Large";
break;
}
}
return headNodeSize;
}
public static ClusterCreateParameters GetCustomCreateParametersPaas()
{
var clusterparams = new ClusterCreateParameters
{
ClusterSizeInNodes = 3,
ClusterType = "Hadoop",
WorkerNodeSize = "Large",
DefaultStorageInfo = GetDefaultAzureStorageInfo(),
OSType = OSType.Windows,
UserName = HttpUser,
Password = HttpPassword,
Location = "East US",
Version = "3.2"
};
var actions = new List<ScriptAction>();
var action = new ScriptAction("action", new Uri("https://uri.com"), "params");
actions.Add(action);
clusterparams.ScriptActions.Add(ClusterNodeType.WorkerNode, actions);
clusterparams.ScriptActions.Add(ClusterNodeType.HeadNode, actions);
return clusterparams;
}
private static void AddDefaultStorageAccountToCoreConfig(string clusterName, ClusterCreateParameters clusterCreateParameters, Dictionary<string, string> coreConfig)
{
string coreConfigDefaultFSKey = "fs.defaultFS";
string coreConfigDefaultFSKeyFor_2_1_Clusters = "fs.default.name";
var defaultStorageAccountKey = (clusterCreateParameters.Version != null && clusterCreateParameters.Version.Equals("2.1"))
? coreConfigDefaultFSKeyFor_2_1_Clusters
: coreConfigDefaultFSKey;
var azureStorageAccountInfo = clusterCreateParameters.DefaultStorageInfo as AzureStorageInfo;
var azureDataLakeStorageInfo = clusterCreateParameters.DefaultStorageInfo as AzureDataLakeStoreInfo;
if(azureStorageAccountInfo != null)
{
if (string.IsNullOrWhiteSpace(azureStorageAccountInfo.StorageContainer))
{
var storageInfoWithContainerName = new AzureStorageInfo(azureStorageAccountInfo.StorageAccountName, azureStorageAccountInfo.StorageAccountKey, clusterName);
clusterCreateParameters.DefaultStorageInfo = storageInfoWithContainerName;
coreConfig[defaultStorageAccountKey] = storageInfoWithContainerName.StorageAccountUri;
}
else
{
coreConfig[defaultStorageAccountKey] = azureStorageAccountInfo.StorageAccountUri;
}
}
else if (azureDataLakeStorageInfo != null)
{
// setup the parameters required for DataLake containers
coreConfig[defaultStorageAccountKey] = "adl://home";
coreConfig["dfs.adls.home.hostname"] = azureDataLakeStorageInfo.StorageAccountName;
coreConfig["dfs.adls.home.mountpoint"] = azureDataLakeStorageInfo.StorageRootPath;
}
}
public static ClusterCreateParameters GetCustomVmSizesCreateParametersIaas()
{
var clusterparams = new ClusterCreateParameters
{
ClusterSizeInNodes = 1,
ClusterType = "HBase",
WorkerNodeSize = "Large",
DefaultStorageInfo = GetDefaultAzureStorageInfo(),
OSType = OSType.Linux,
UserName = HttpUser,
Password = HttpPassword,
Location = "East US",
SshUserName = SshUser,
SshPassword = SshPassword,
Version = "3.2",
HeadNodeSize = "ExtraLarge",
ZookeeperNodeSize = "Large",
};
return clusterparams;
}
public NewAzureHDInsightClusterCommand()
{
parameters = new ClusterCreateParameters();
AdditionalStorageAccounts = new Dictionary<string, string>();
Configurations = new Dictionary<string, Dictionary<string, string>>();
ScriptActions = new Dictionary<ClusterNodeType, List<AzureHDInsightScriptAction>>();
}
internal static string GetHeadNodeSize(ClusterCreateParameters clusterCreateParameters)
{
string headNodeSize;
if (clusterCreateParameters.HeadNodeSize != null)
{
headNodeSize = clusterCreateParameters.HeadNodeSize;
}
else
{
if (! HeadNodeDefaultSizes.TryGetValue(clusterCreateParameters.ClusterType, out headNodeSize))
{
headNodeSize = "Large";
}
}
return headNodeSize;
}
