public static Role ExtractRole(string nodeType, ComputeProfile computeProfile)
{
return(computeProfile?.Roles?.FirstOrDefault(role => role.Name.Equals(nodeType.ToLower())));
}
public override void ExecuteCmdlet()
{
foreach (var component in ComponentVersion.Where(component => !clusterComponentVersion.ContainsKey(component.Key)))
{
clusterComponentVersion.Add(component.Key, component.Value);
}
// Construct Configurations
foreach (var config in Configurations.Where(config => !clusterConfigurations.ContainsKey(config.Key)))
{
clusterConfigurations.Add(config.Key, config.Value);
}
// Add cluster username/password to gateway config.
ClusterCreateHelper.AddClusterCredentialToGatewayConfig(HttpCredential, clusterConfigurations);
// Construct OS Profile
OsProfile osProfile = ClusterCreateHelper.CreateOsProfile(SshCredential, SshPublicKey);
// Construct Virtual Network Profile
VirtualNetworkProfile vnetProfile = ClusterCreateHelper.CreateVirtualNetworkProfile(VirtualNetworkId, SubnetName);
// Handle storage account
StorageProfile storageProfile = new StorageProfile()
{
Storageaccounts = new List <StorageAccount> {
}
};
if (StorageAccountType == null || StorageAccountType == StorageType.AzureStorage)
{
var azureStorageAccount = ClusterCreateHelper.CreateAzureStorageAccount(ClusterName, StorageAccountResourceId, StorageAccountKey, StorageContainer, this.DefaultContext.Environment.StorageEndpointSuffix);
storageProfile.Storageaccounts.Add(azureStorageAccount);
}
else if (StorageAccountType == StorageType.AzureDataLakeStore)
{
ClusterCreateHelper.AddAzureDataLakeStorageGen1ToCoreConfig(StorageAccountResourceId, StorageRootPath, this.DefaultContext.Environment.AzureDataLakeStoreFileSystemEndpointSuffix, clusterConfigurations);
}
else if (StorageAccountType == StorageType.AzureDataLakeStorageGen2)
{
var adlsgen2Account = ClusterCreateHelper.CreateAdlsGen2StorageAccount(ClusterName, StorageAccountResourceId, StorageAccountKey, StorageFileSystem, StorageAccountManagedIdentity, this.DefaultContext.Environment.StorageEndpointSuffix);
storageProfile.Storageaccounts.Add(adlsgen2Account);
}
// Handle additional storage accounts
foreach (
var storageAccount in
AdditionalStorageAccounts.Where(
storageAccount => !clusterAdditionalStorageAccounts.ContainsKey(storageAccount.Key)))
{
clusterAdditionalStorageAccounts.Add(storageAccount.Key, storageAccount.Value);
}
ClusterCreateHelper.AddAdditionalStorageAccountsToCoreConfig(clusterAdditionalStorageAccounts, clusterConfigurations);
// Handle script action
foreach (var action in ScriptActions.Where(action => clusterScriptActions.ContainsKey(action.Key)))
{
clusterScriptActions.Add(action.Key,
action.Value.Select(a => a.GetScriptActionFromPSModel()).ToList());
}
// Handle metastore
if (OozieMetastore != null)
{
ClusterCreateHelper.AddOozieMetastoreToConfigurations(OozieMetastore, clusterConfigurations);
}
if (HiveMetastore != null)
{
ClusterCreateHelper.AddHiveMetastoreToConfigurations(HiveMetastore, clusterConfigurations);
}
// Handle ADLSGen1 identity
if (!string.IsNullOrEmpty(CertificatePassword))
{
if (!string.IsNullOrEmpty(CertificateFilePath))
{
CertificateFileContents = File.ReadAllBytes(CertificateFilePath);
}
ClusterCreateHelper.AddDataLakeStorageGen1IdentityToIdentityConfig(
GetApplicationId(ApplicationId), GetTenantId(AadTenantId), CertificateFileContents, CertificatePassword, clusterConfigurations,
this.DefaultContext.Environment.ActiveDirectoryAuthority, this.DefaultContext.Environment.DataLakeEndpointResourceId);
}
// Handle Kafka Rest Proxy
KafkaRestProperties kafkaRestProperties = null;
if (KafkaClientGroupId != null && KafkaClientGroupName != null)
{
kafkaRestProperties = new KafkaRestProperties()
{
ClientGroupInfo = new ClientGroupInfo(KafkaClientGroupName, KafkaClientGroupId)
};
}
// Compute profile contains headnode, workernode, zookeepernode, edgenode, kafkamanagementnode, idbrokernode, etc.
ComputeProfile computeProfile = ClusterCreateHelper.CreateComputeProfile(osProfile, vnetProfile, clusterScriptActions, ClusterType, ClusterSizeInNodes, HeadNodeSize, WorkerNodeSize, ZookeeperNodeSize, EdgeNodeSize, KafkaManagementNodeSize, EnableIDBroker.IsPresent);
// Handle SecurityProfile
SecurityProfile securityProfile = ClusterCreateHelper.ConvertAzureHDInsightSecurityProfileToSecurityProfile(SecurityProfile, AssignedIdentity);
// Handle DisksPerWorkerNode feature
Role workerNode = Utils.ExtractRole(ClusterNodeType.WorkerNode.ToString(), computeProfile);
if (DisksPerWorkerNode > 0)
{
workerNode.DataDisksGroups = new List <DataDisksGroups>()
{
new DataDisksGroups()
{
DisksPerNode = DisksPerWorkerNode
}
};
}
// Handle ClusterIdentity
ClusterIdentity clusterIdentity = null;
if (AssignedIdentity != null || StorageAccountManagedIdentity != null)
{
clusterIdentity = new ClusterIdentity
{
Type = ResourceIdentityType.UserAssigned,
UserAssignedIdentities = new Dictionary <string, ClusterIdentityUserAssignedIdentitiesValue>()
};
if (AssignedIdentity != null)
{
clusterIdentity.UserAssignedIdentities.Add(AssignedIdentity, new ClusterIdentityUserAssignedIdentitiesValue());
}
if (StorageAccountManagedIdentity != null)
{
clusterIdentity.UserAssignedIdentities.Add(StorageAccountManagedIdentity, new ClusterIdentityUserAssignedIdentitiesValue());
}
}
// Handle CMK feature
DiskEncryptionProperties diskEncryptionProperties = null;
if (EncryptionKeyName != null && EncryptionKeyVersion != null && EncryptionVaultUri != null)
{
diskEncryptionProperties = new DiskEncryptionProperties()
{
KeyName = EncryptionKeyName,
KeyVersion = EncryptionKeyVersion,
VaultUri = EncryptionVaultUri,
EncryptionAlgorithm = EncryptionAlgorithm != null ? EncryptionAlgorithm : JsonWebKeyEncryptionAlgorithm.RSAOAEP,
MsiResourceId = AssignedIdentity
};
}
// Handle encryption at host feature
if (EncryptionAtHost != null)
{
if (diskEncryptionProperties != null)
{
diskEncryptionProperties.EncryptionAtHost = EncryptionAtHost;
}
else
{
diskEncryptionProperties = new DiskEncryptionProperties()
{
EncryptionAtHost = EncryptionAtHost
};
}
}
// Handle autoscale featurer
Autoscale autoscaleParameter = null;
if (AutoscaleConfiguration != null)
{
autoscaleParameter = AutoscaleConfiguration.ToAutoscale();
workerNode.AutoscaleConfiguration = autoscaleParameter;
}
// Construct cluster create parameter
ClusterCreateParametersExtended createParams = new ClusterCreateParametersExtended
{
Location = Location,
//Tags = Tags, //To Do add this Tags parameter
Properties = new ClusterCreateProperties
{
Tier = ClusterTier,
ClusterDefinition = new ClusterDefinition
{
Kind = ClusterType ?? "Hadoop",
ComponentVersion = clusterComponentVersion,
Configurations = clusterConfigurations
},
ClusterVersion = Version ?? "default",
KafkaRestProperties = kafkaRestProperties,
ComputeProfile = computeProfile,
OsType = OSType,
SecurityProfile = securityProfile,
StorageProfile = storageProfile,
DiskEncryptionProperties = diskEncryptionProperties,
//handle Encryption In Transit feature
EncryptionInTransitProperties = EncryptionInTransit != null ? new EncryptionInTransitProperties()
{
IsEncryptionInTransitEnabled = EncryptionInTransit
} : null,
MinSupportedTlsVersion = MinSupportedTlsVersion
},
Identity = clusterIdentity
};
var cluster = HDInsightManagementClient.CreateCluster(ResourceGroupName, ClusterName, createParams);
if (cluster != null)
{
WriteObject(new AzureHDInsightCluster(cluster));
}
}
public AzureHDInsightComputeProfile(ComputeProfile computeProfile)
{
Roles = computeProfile.Roles?.Select(role => new AzureHDInsightRole(role)).ToList();
}