public static ClusterCreateParametersExtended GetExtendedClusterCreateParameters(string clusterName, ClusterCreateParameters createParameters)
{
//Deep copy so the createParameters object isn't touched.
ClusterCreateParameters clusterCreateParameters = new ClusterCreateParameters(createParameters);
//Convert to extended spec.
ClusterCreateParametersExtended extendedParams = new ClusterCreateParametersExtended
{
Location = clusterCreateParameters.Location,
Tags = clusterCreateParameters.Tags,
Properties = new ClusterCreateProperties
{
ClusterDefinition = new ClusterDefinition
{
Kind = clusterCreateParameters.ClusterType,
ComponentVersion = clusterCreateParameters.ComponentVersion,
Configurations = GetConfigurations(clusterName, clusterCreateParameters)
},
Tier = clusterCreateParameters.ClusterTier,
ClusterVersion = clusterCreateParameters.Version,
ComputeProfile = new ComputeProfile
{
Roles = GetRoleCollection(clusterCreateParameters)
},
OsType = OSType.Linux,
SecurityProfile = clusterCreateParameters.SecurityProfile,
StorageProfile = GetStorageProfile(clusterCreateParameters),
DiskEncryptionProperties = clusterCreateParameters.DiskEncryptionProperties
},
Identity = clusterCreateParameters.ClusterIdentity
};
return(extendedParams);
}
public static void ValidateSpecConversion(ClusterCreateParameters createParams)
{
//Convert spec.
ClusterCreateParametersExtended extendedParams = CreateParametersConverter.GetExtendedClusterCreateParameters("testCluster", createParams);
//Validate properties.
Assert.Equal(createParams.Tags, extendedParams.Tags);
Assert.Equal(createParams.Location, extendedParams.Location);
Assert.Equal(createParams.ClusterType, extendedParams.Properties.ClusterDefinition.Kind);
Assert.Equal(createParams.ComponentVersion, extendedParams.Properties.ClusterDefinition.ComponentVersion);
Assert.Equal(createParams.ClusterTier, extendedParams.Properties.Tier);
Assert.Equal(createParams.Version, extendedParams.Properties.ClusterVersion);
Assert.Equal(OSType.Linux, extendedParams.Properties.OsType);
Assert.Equal(createParams.SecurityProfile, extendedParams.Properties.SecurityProfile);
ValidateStorageProfile(createParams.DefaultStorageInfo, extendedParams.Properties.StorageProfile);
//Validate configurations.
Dictionary <string, Dictionary <string, string> > configurations = extendedParams.Properties.ClusterDefinition.Configurations as Dictionary <string, Dictionary <string, string> >;
Assert.NotNull(configurations);
ValidateStorageConfigurations(configurations, createParams.DefaultStorageInfo, createParams.AdditionalStorageAccounts);
ValidateClusterCredentialConfigurations(configurations);
ValidateAdlConfigurations(configurations, createParams.Principal);
ValidateHiveMetastoreConfigurations(configurations, createParams.HiveMetastore);
ValidateOozieMetastoreConfigurations(configurations, createParams.OozieMetastore);
//Validate roles.
ValidateRoles(extendedParams.Properties.ComputeProfile.Roles, createParams);
}
public static ClusterCreateParametersExtended AddConfigurations(ClusterCreateParametersExtended cluster, string configurationKey, Dictionary <string, string> configs)
{
string configurations = cluster.Properties.ClusterDefinition.Configurations;
var config = JsonConvert.DeserializeObject <Dictionary <string, Dictionary <String, string> > >(configurations);
config.Add(configurationKey, configs);
var serializedConfig = JsonConvert.SerializeObject(config);
cluster.Properties.ClusterDefinition.Configurations = serializedConfig;
return(cluster);
}
private void AssertContainerMatchesNameProvided(ClusterCreateParametersExtended createParamsExtended, string name)
{
Dictionary <string, string> coresiteConfig;
if (JsonConvert.DeserializeObject <Dictionary <string, Dictionary <string, string> > >(createParamsExtended.Properties.ClusterDefinition.Configurations).TryGetValue("core-site", out coresiteConfig))
{
string value;
coresiteConfig.TryGetValue("fs.defaultFS", out value);
Assert.True(value.StartsWith("wasb://" + name + "@"), "Container does not match the name provided");
}
else
{
Assert.True(false, "Deserialization error");
}
}
/// <summary>
/// Validate cluster
/// </summary>
/// <param name="expectedClustername"></param>
/// <param name="expectedParameters"></param>
/// <param name="actualCluster"></param>
public static void ValidateCluster(string expectedClustername, ClusterCreateParametersExtended expectedParameters, Cluster actualCluster)
{
Assert.Equal(expectedClustername, actualCluster.Name);
Assert.Equal(expectedParameters.Properties.Tier, actualCluster.Properties.Tier);
Assert.NotNull(actualCluster.Etag);
Assert.EndsWith(expectedClustername, actualCluster.Id);
Assert.Equal("Running", actualCluster.Properties.ClusterState);
Assert.Equal("Microsoft.HDInsight/clusters", actualCluster.Type);
Assert.Equal(expectedParameters.Location, actualCluster.Location);
Assert.Equal(expectedParameters.Tags, actualCluster.Tags);
Assert.Equal(1, actualCluster.Properties.ConnectivityEndpoints.Count(c => c.Name.Equals("HTTPS", StringComparison.OrdinalIgnoreCase)));
Assert.Equal(1, actualCluster.Properties.ConnectivityEndpoints.Count(c => c.Name.Equals("SSH", StringComparison.OrdinalIgnoreCase)));
Assert.Equal(expectedParameters.Properties.OsType, actualCluster.Properties.OsType);
Assert.Null(actualCluster.Properties.Errors);
Assert.Equal(HDInsightClusterProvisioningState.Succeeded, actualCluster.Properties.ProvisioningState);
Assert.Equal(expectedParameters.Properties.ClusterDefinition.Kind, actualCluster.Properties.ClusterDefinition.Kind);
Assert.Equal(expectedParameters.Properties.ClusterVersion, actualCluster.Properties.ClusterVersion.Substring(0, 3));
Assert.Null(actualCluster.Properties.ClusterDefinition.Configurations);
}
/// <summary>
/// Create cluster create parameters for ADLS Gen2 relevant tests
/// </summary>
/// <param name="commonData"></param>
/// <param name="storageAccountName"></param>
/// <param name="storageResourceId"></param>
/// <param name="msiResourceId"></param>
/// <param name="createParams"></param>
/// <returns></returns>
public static ClusterCreateParametersExtended PrepareClusterCreateParamsForADLSv2(
this CommonTestFixture commonData,
string storageAccountName,
string storageResourceId,
string msiResourceId,
ClusterCreateParametersExtended createParams = null)
{
var createParamsForADLSv2 = createParams ?? commonData.PrepareClusterCreateParams();
bool isDefault = !createParamsForADLSv2.Properties.StorageProfile.Storageaccounts.Any();
createParamsForADLSv2.Properties.StorageProfile.Storageaccounts.Add(
new HDInsightStorageAccount
{
Name = storageAccountName + commonData.DfsEndpointSuffix,
IsDefault = isDefault,
FileSystem = commonData.ContainerName.ToLowerInvariant(),
ResourceId = storageResourceId,
MsiResourceId = msiResourceId
}
);
var identity = new ClusterIdentity
{
Type = ResourceIdentityType.UserAssigned,
UserAssignedIdentities = new Dictionary <string, ClusterIdentityUserAssignedIdentitiesValue>
{
{ msiResourceId, new ClusterIdentityUserAssignedIdentitiesValue() }
}
};
if (createParamsForADLSv2.Identity == null)
{
createParamsForADLSv2.Identity = identity;
}
else
{
// At this point, only user-assigned managed identity is supported by HDInsight.
// So identity type is not checked.
createParamsForADLSv2.Identity.UserAssignedIdentities.Union(identity.UserAssignedIdentities);
}
return(createParamsForADLSv2);
}
private void ResetClusterIdentity(ClusterCreateParametersExtended createParams, string aadAuthority, string dataLakeAudience)
{
var configuation = (Dictionary <string, Dictionary <string, string> >)createParams.Properties.ClusterDefinition.Configurations;
Dictionary <string, string> clusterIdentity;
if (!configuation.TryGetValue("clusterIdentity", out clusterIdentity))
{
return;
}
clusterIdentity["clusterIdentity.resourceUri"] = dataLakeAudience;
string aadTenantIdWithUrl;
clusterIdentity.TryGetValue("clusterIdentity.aadTenantId", out aadTenantIdWithUrl);
const string defaultPubliCloudAadAuthority = "https://login.windows.net/";
string newAadTenantIdWithUrl = aadTenantIdWithUrl?.Replace(defaultPubliCloudAadAuthority, aadAuthority);
clusterIdentity["clusterIdentity.aadTenantId"] = newAadTenantIdWithUrl;
}
public void CanConvertHadoopClusterWithCustomMetastoresToMarchSpecWithConfigurations()
{
ClusterCreateParameters createParams = GetClusterCreateParamsWithMinRequiredValues();
createParams.HiveMetastore = new Metastore("server.database.windows.net", "hiveDb", "username", "password");
createParams.OozieMetastore = new Metastore("server.database.windows.net", "oozieDb", "username", "password");
Dictionary <string, string> testConfig = new Dictionary <string, string>
{
{ "key", "value" }
};
createParams.Configurations.Add(ConfigurationKey.HiveSite, testConfig);
createParams.Configurations.Add(ConfigurationKey.HiveEnv, testConfig);
createParams.Configurations.Add(ConfigurationKey.OozieEnv, testConfig);
createParams.Configurations.Add(ConfigurationKey.OozieSite, testConfig);
ClusterCreateParametersExtended extendedParams = CreateParametersConverter.GetExtendedClusterCreateParameters("testCluster", createParams);
Dictionary <string, Dictionary <string, string> > configurations = extendedParams.Properties.ClusterDefinition.Configurations as Dictionary <string, Dictionary <string, string> >;
Assert.True(configurations.ContainsKey(ConfigurationKey.HiveSite));
Dictionary <string, string> hiveSiteConfig = configurations[ConfigurationKey.HiveSite];
Assert.Equal(5, hiveSiteConfig.Count);
}
public override void ExecuteCmdlet()
{
foreach (var component in ComponentVersion.Where(component => !clusterComponentVersion.ContainsKey(component.Key)))
{
clusterComponentVersion.Add(component.Key, component.Value);
}
// Construct Configurations
foreach (var config in Configurations.Where(config => !clusterConfigurations.ContainsKey(config.Key)))
{
clusterConfigurations.Add(config.Key, config.Value);
}
// Add cluster username/password to gateway config.
ClusterCreateHelper.AddClusterCredentialToGatewayConfig(HttpCredential, clusterConfigurations);
// Construct OS Profile
OsProfile osProfile = ClusterCreateHelper.CreateOsProfile(SshCredential, SshPublicKey);
// Construct Virtual Network Profile
VirtualNetworkProfile vnetProfile = ClusterCreateHelper.CreateVirtualNetworkProfile(VirtualNetworkId, SubnetName);
// Handle storage account
StorageProfile storageProfile = new StorageProfile()
{
Storageaccounts = new List <StorageAccount> {
}
};
if (StorageAccountType == null || StorageAccountType == StorageType.AzureStorage)
{
var azureStorageAccount = ClusterCreateHelper.CreateAzureStorageAccount(ClusterName, StorageAccountResourceId, StorageAccountKey, StorageContainer, this.DefaultContext.Environment.StorageEndpointSuffix);
storageProfile.Storageaccounts.Add(azureStorageAccount);
}
else if (StorageAccountType == StorageType.AzureDataLakeStore)
{
ClusterCreateHelper.AddAzureDataLakeStorageGen1ToCoreConfig(StorageAccountResourceId, StorageRootPath, this.DefaultContext.Environment.AzureDataLakeStoreFileSystemEndpointSuffix, clusterConfigurations);
}
else if (StorageAccountType == StorageType.AzureDataLakeStorageGen2)
{
var adlsgen2Account = ClusterCreateHelper.CreateAdlsGen2StorageAccount(ClusterName, StorageAccountResourceId, StorageAccountKey, StorageFileSystem, StorageAccountManagedIdentity, this.DefaultContext.Environment.StorageEndpointSuffix);
storageProfile.Storageaccounts.Add(adlsgen2Account);
}
// Handle additional storage accounts
foreach (
var storageAccount in
AdditionalStorageAccounts.Where(
storageAccount => !clusterAdditionalStorageAccounts.ContainsKey(storageAccount.Key)))
{
clusterAdditionalStorageAccounts.Add(storageAccount.Key, storageAccount.Value);
}
ClusterCreateHelper.AddAdditionalStorageAccountsToCoreConfig(clusterAdditionalStorageAccounts, clusterConfigurations);
// Handle script action
foreach (var action in ScriptActions.Where(action => clusterScriptActions.ContainsKey(action.Key)))
{
clusterScriptActions.Add(action.Key,
action.Value.Select(a => a.GetScriptActionFromPSModel()).ToList());
}
// Handle metastore
if (OozieMetastore != null)
{
ClusterCreateHelper.AddOozieMetastoreToConfigurations(OozieMetastore, clusterConfigurations);
}
if (HiveMetastore != null)
{
ClusterCreateHelper.AddHiveMetastoreToConfigurations(HiveMetastore, clusterConfigurations);
}
// Handle ADLSGen1 identity
if (!string.IsNullOrEmpty(CertificatePassword))
{
if (!string.IsNullOrEmpty(CertificateFilePath))
{
CertificateFileContents = File.ReadAllBytes(CertificateFilePath);
}
ClusterCreateHelper.AddDataLakeStorageGen1IdentityToIdentityConfig(
GetApplicationId(ApplicationId), GetTenantId(AadTenantId), CertificateFileContents, CertificatePassword, clusterConfigurations,
this.DefaultContext.Environment.ActiveDirectoryAuthority, this.DefaultContext.Environment.DataLakeEndpointResourceId);
}
// Handle Kafka Rest Proxy
KafkaRestProperties kafkaRestProperties = null;
if (KafkaClientGroupId != null && KafkaClientGroupName != null)
{
kafkaRestProperties = new KafkaRestProperties()
{
ClientGroupInfo = new ClientGroupInfo(KafkaClientGroupName, KafkaClientGroupId)
};
}
// Compute profile contains headnode, workernode, zookeepernode, edgenode, kafkamanagementnode, idbrokernode, etc.
ComputeProfile computeProfile = ClusterCreateHelper.CreateComputeProfile(osProfile, vnetProfile, clusterScriptActions, ClusterType, ClusterSizeInNodes, HeadNodeSize, WorkerNodeSize, ZookeeperNodeSize, EdgeNodeSize, KafkaManagementNodeSize, EnableIDBroker.IsPresent);
// Handle SecurityProfile
SecurityProfile securityProfile = ClusterCreateHelper.ConvertAzureHDInsightSecurityProfileToSecurityProfile(SecurityProfile, AssignedIdentity);
// Handle DisksPerWorkerNode feature
Role workerNode = Utils.ExtractRole(ClusterNodeType.WorkerNode.ToString(), computeProfile);
if (DisksPerWorkerNode > 0)
{
workerNode.DataDisksGroups = new List <DataDisksGroups>()
{
new DataDisksGroups()
{
DisksPerNode = DisksPerWorkerNode
}
};
}
// Handle ClusterIdentity
ClusterIdentity clusterIdentity = null;
if (AssignedIdentity != null || StorageAccountManagedIdentity != null)
{
clusterIdentity = new ClusterIdentity
{
Type = ResourceIdentityType.UserAssigned,
UserAssignedIdentities = new Dictionary <string, ClusterIdentityUserAssignedIdentitiesValue>()
};
if (AssignedIdentity != null)
{
clusterIdentity.UserAssignedIdentities.Add(AssignedIdentity, new ClusterIdentityUserAssignedIdentitiesValue());
}
if (StorageAccountManagedIdentity != null)
{
clusterIdentity.UserAssignedIdentities.Add(StorageAccountManagedIdentity, new ClusterIdentityUserAssignedIdentitiesValue());
}
}
// Handle CMK feature
DiskEncryptionProperties diskEncryptionProperties = null;
if (EncryptionKeyName != null && EncryptionKeyVersion != null && EncryptionVaultUri != null)
{
diskEncryptionProperties = new DiskEncryptionProperties()
{
KeyName = EncryptionKeyName,
KeyVersion = EncryptionKeyVersion,
VaultUri = EncryptionVaultUri,
EncryptionAlgorithm = EncryptionAlgorithm != null ? EncryptionAlgorithm : JsonWebKeyEncryptionAlgorithm.RSAOAEP,
MsiResourceId = AssignedIdentity
};
}
// Handle encryption at host feature
if (EncryptionAtHost != null)
{
if (diskEncryptionProperties != null)
{
diskEncryptionProperties.EncryptionAtHost = EncryptionAtHost;
}
else
{
diskEncryptionProperties = new DiskEncryptionProperties()
{
EncryptionAtHost = EncryptionAtHost
};
}
}
// Handle autoscale featurer
Autoscale autoscaleParameter = null;
if (AutoscaleConfiguration != null)
{
autoscaleParameter = AutoscaleConfiguration.ToAutoscale();
workerNode.AutoscaleConfiguration = autoscaleParameter;
}
// Construct cluster create parameter
ClusterCreateParametersExtended createParams = new ClusterCreateParametersExtended
{
Location = Location,
//Tags = Tags, //To Do add this Tags parameter
Properties = new ClusterCreateProperties
{
Tier = ClusterTier,
ClusterDefinition = new ClusterDefinition
{
Kind = ClusterType ?? "Hadoop",
ComponentVersion = clusterComponentVersion,
Configurations = clusterConfigurations
},
ClusterVersion = Version ?? "default",
KafkaRestProperties = kafkaRestProperties,
ComputeProfile = computeProfile,
OsType = OSType,
SecurityProfile = securityProfile,
StorageProfile = storageProfile,
DiskEncryptionProperties = diskEncryptionProperties,
//handle Encryption In Transit feature
EncryptionInTransitProperties = EncryptionInTransit != null ? new EncryptionInTransitProperties()
{
IsEncryptionInTransitEnabled = EncryptionInTransit
} : null,
MinSupportedTlsVersion = MinSupportedTlsVersion
},
Identity = clusterIdentity
};
var cluster = HDInsightManagementClient.CreateCluster(ResourceGroupName, ClusterName, createParams);
if (cluster != null)
{
WriteObject(new AzureHDInsightCluster(cluster));
}
}
public virtual Cluster CreateCluster(string resourceGroupName, string clusterName, ClusterCreateParametersExtended createParams)
{
return(HdInsightManagementClient.Clusters.Create(resourceGroupName, clusterName, createParams));
}
/// <summary>
/// Creates a new HDInsight cluster with the specified parameters.
/// </summary>
/// <param name='operations'>
/// The operations group for this extension method.
/// </param>
/// <param name='resourceGroupName'>
/// The name of the resource group.
/// </param>
/// <param name='clusterName'>
/// The name of the cluster.
/// </param>
/// <param name='parameters'>
/// The cluster create request.
/// </param>
/// <param name='cancellationToken'>
/// The cancellation token.
/// </param>
public static async Task <Cluster> CreateAsync(this IClustersOperations operations, string resourceGroupName, string clusterName, ClusterCreateParametersExtended parameters, CancellationToken cancellationToken = default(CancellationToken))
{
using (var _result = await operations.CreateWithHttpMessagesAsync(resourceGroupName, clusterName, parameters, null, cancellationToken).ConfigureAwait(false))
{
return(_result.Body);
}
}
/// <summary>
/// Creates a new HDInsight cluster with the specified parameters.
/// </summary>
/// <param name='operations'>
/// The operations group for this extension method.
/// </param>
/// <param name='resourceGroupName'>
/// The name of the resource group.
/// </param>
/// <param name='clusterName'>
/// The name of the cluster.
/// </param>
/// <param name='parameters'>
/// The cluster create request.
/// </param>
public static Cluster BeginCreate(this IClustersOperations operations, string resourceGroupName, string clusterName, ClusterCreateParametersExtended parameters)
{
return(operations.BeginCreateAsync(resourceGroupName, clusterName, parameters).GetAwaiter().GetResult());
}
internal static void ClassMain(string[] args)
{
// Authentication
var credentials = SdkContext.AzureCredentialsFactory
.FromServicePrincipal(
Configurations.ClientId,
Configurations.ClientSecret,
Configurations.TenantId,
AzureEnvironment.AzureGlobalCloud);
var client = new HDInsightManagementClient(credentials)
{
SubscriptionId = Configurations.SubscriptionId
};
string clusterName = Configurations.ClusterName;
// Prepare cluster create parameters
var createParams = new ClusterCreateParametersExtended
{
Location = Configurations.Location,
Properties = new ClusterCreateProperties
{
ClusterVersion = "3.6",
OsType = OSType.Linux,
Tier = Tier.Standard,
ClusterDefinition = new ClusterDefinition
{
Kind = "Spark",
Configurations = new Dictionary <string, Dictionary <string, string> >()
{
{ "gateway", new Dictionary <string, string>
{
{ "restAuthCredential.isEnabled", "true" },
{ "restAuthCredential.username", Configurations.ClusterLoginUserName },
{ "restAuthCredential.password", Configurations.Password }
} }
}
},
ComputeProfile = new ComputeProfile
{
Roles = new List <Role>
{
new Role
{
Name = "headnode",
TargetInstanceCount = 2,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
Username = Configurations.SshUserName,
Password = Configurations.Password
}
}
},
new Role
{
Name = "workernode",
TargetInstanceCount = 3,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
Username = Configurations.SshUserName,
Password = Configurations.Password
}
}
}
}
},
StorageProfile = new StorageProfile
{
Storageaccounts = new List <StorageAccount>
{
new StorageAccount
{
Name = Configurations.StorageAccountName + Configurations.BlobEndpointSuffix,
Key = Configurations.StorageAccountKey,
Container = Configurations.ContainerName.ToLowerInvariant(),
IsDefault = true
}
}
}
}
};
Console.WriteLine($"Start to create HDInsight Spark cluster {clusterName}");
var cluster = client.Clusters.Create(Configurations.ResourceGroupName, clusterName, createParams);
Console.WriteLine($"Finish creating HDInsight Spark cluster {clusterName}");
}
internal static void ClassMain(string[] args)
{
// Authentication
var credentials = SdkContext.AzureCredentialsFactory
.FromServicePrincipal(
Configurations.ClientId,
Configurations.ClientSecret,
Configurations.TenantId,
AzureEnvironment.AzureGlobalCloud);
var client = new HDInsightManagementClient(credentials)
{
SubscriptionId = Configurations.SubscriptionId
};
string clusterName = Configurations.ClusterName;
// Parse AAD-DS DNS Domain name from resource id
string aaddsDnsDomainName = Configurations.AaddsResourceId.Split('/').Last();
// Prepare cluster create parameters
var createParams = new ClusterCreateParametersExtended
{
Location = Configurations.Location,
Properties = new ClusterCreateProperties
{
ClusterVersion = "3.6",
OsType = OSType.Linux,
Tier = Tier.Premium,
ClusterDefinition = new ClusterDefinition
{
Kind = "Spark",
Configurations = new Dictionary <string, Dictionary <string, string> >()
{
{ "gateway", new Dictionary <string, string>
{
{ "restAuthCredential.isEnabled", "true" },
{ "restAuthCredential.username", Configurations.ClusterLoginUserName },
{ "restAuthCredential.password", Configurations.Password }
} }
}
},
ComputeProfile = new ComputeProfile
{
Roles = new List <Role>
{
new Role
{
Name = "headnode",
TargetInstanceCount = 2,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
Username = Configurations.SshUserName,
Password = Configurations.Password
}
},
VirtualNetworkProfile = new VirtualNetworkProfile
{
Id = Configurations.VirtualNetworkResourceId,
Subnet = string.Format("{0}/subnets/{1}", Configurations.VirtualNetworkResourceId, Configurations.SubnetName)
}
},
new Role
{
Name = "workernode",
TargetInstanceCount = 3,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
Username = Configurations.SshUserName,
Password = Configurations.Password
}
},
VirtualNetworkProfile = new VirtualNetworkProfile
{
Id = Configurations.VirtualNetworkResourceId,
Subnet = string.Format("{0}/subnets/{1}", Configurations.VirtualNetworkResourceId, Configurations.SubnetName)
}
}
}
},
StorageProfile = new StorageProfile
{
Storageaccounts = new List <StorageAccount>
{
new StorageAccount
{
Name = Configurations.StorageAccountName + Configurations.BlobEndpointSuffix,
Key = Configurations.StorageAccountKey,
Container = Configurations.ContainerName.ToLowerInvariant(),
IsDefault = true
}
}
},
SecurityProfile = new SecurityProfile
{
DirectoryType = DirectoryType.ActiveDirectory,
LdapsUrls = new[] { Configurations.LdapsUrl },
DomainUsername = Configurations.DomainUserName,
Domain = aaddsDnsDomainName,
ClusterUsersGroupDNs = new[] { Configurations.ClusterAccessGroup },
AaddsResourceId = Configurations.AaddsResourceId,
MsiResourceId = Configurations.ManagedIdentityResourceId
}
},
Identity = new ClusterIdentity
{
Type = ResourceIdentityType.UserAssigned,
UserAssignedIdentities = new Dictionary <string, ClusterIdentityUserAssignedIdentitiesValue>
{
{ Configurations.ManagedIdentityResourceId, new ClusterIdentityUserAssignedIdentitiesValue() }
}
}
};
Console.WriteLine($"Start to create HDInsight Spark cluster {clusterName} with Enterprise Security Package");
client.Clusters.Create(Configurations.ResourceGroupName, clusterName, createParams);
Console.WriteLine($"Finish creating HDInsight Spark cluster {clusterName} with Enterprise Security Package");
}
public static ClusterCreateParametersExtended GetIaasClusterSpec()
{
var cluster = new ClusterCreateParametersExtended
{
Location = "West US",
Properties = new ClusterCreateProperties
{
ClusterDefinition = new ClusterDefinition
{
ClusterType = "Hadoop"
},
ClusterVersion = "3.2",
OperatingSystemType = OSType.Linux
}
};
var coreConfigs = new Dictionary <string, string>
{
{ "fs.defaultFS", string.Format("wasb://{0}@{1}", DefaultContainer, StorageAccountName) },
{
string.Format("fs.azure.account.key.{0}", StorageAccountName),
StorageAccountKey
}
};
var gatewayConfigs = new Dictionary <string, string>
{
{ "restAuthCredential.isEnabled", "true" },
{ "restAuthCredential.username", HttpUser },
{ "restAuthCredential.password", HttpPassword }
};
var configurations = new Dictionary <string, Dictionary <string, string> >
{
{ "core-site", coreConfigs },
{ "gateway", gatewayConfigs }
};
var serializedConfig = JsonConvert.SerializeObject(configurations);
cluster.Properties.ClusterDefinition.Configurations = serializedConfig;
cluster.Tags.Add("tag1", "value1");
cluster.Tags.Add("tag2", "value2");
var sshPublicKeys = new List <SshPublicKey>();
var sshPublicKey = new SshPublicKey
{
CertificateData =
string.Format("ssh-rsa {0}", SshKey)
};
sshPublicKeys.Add(sshPublicKey);
var headNode = new Role
{
Name = "headnode",
TargetInstanceCount = 1,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
UserName = "******",
SshProfile = new SshProfile
{
SshPublicKeys = sshPublicKeys
}
}
},
VirtualNetworkProfile = new VirtualNetworkProfile
{
Id = "vnetid",
SubnetName = "subnetname"
}
};
var workerNode = new Role
{
Name = "workernode",
TargetInstanceCount = 1,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
UserName = "******",
SshProfile = new SshProfile
{
SshPublicKeys = sshPublicKeys
}
}
}
};
cluster.Properties.ComputeProfile = new ComputeProfile();
cluster.Properties.ComputeProfile.Roles.Add(headNode);
cluster.Properties.ComputeProfile.Roles.Add(workerNode);
return(cluster);
}
public static ClusterCreateParametersExtended GetPaasClusterSpec()
{
var cluster = new ClusterCreateParametersExtended
{
Location = "East US",
Properties = new ClusterCreateProperties
{
ClusterDefinition = new ClusterDefinition
{
ClusterType = "Hadoop"
},
ClusterVersion = "3.1",
OperatingSystemType = OSType.Windows
}
};
var coreConfigs = new Dictionary <string, string>
{
{ "fs.defaultFS", string.Format("wasb://{0}@{1}", DefaultContainer, StorageAccountName) },
{
string.Format("fs.azure.account.key.{0}", StorageAccountName), StorageAccountKey
}
};
var gatewayConfigs = new Dictionary <string, string>
{
{ "restAuthCredential.isEnabled", "true" },
{ "restAuthCredential.username", HttpUser },
{ "restAuthCredential.password", HttpPassword }
};
var configurations = new Dictionary <string, Dictionary <string, string> >
{
{ "core-site", coreConfigs },
{ "gateway", gatewayConfigs }
};
var serializedConfig = JsonConvert.SerializeObject(configurations);
cluster.Properties.ClusterDefinition.Configurations = serializedConfig;
cluster.Tags.Add("tag1", "value1");
cluster.Tags.Add("tag2", "value2");
var headNode = new Role
{
Name = "headnode",
TargetInstanceCount = 2,
HardwareProfile = new HardwareProfile
{
VmSize = "ExtraLarge"
},
OsProfile = new OsProfile
{
WindowsOperatingSystemProfile = new WindowsOperatingSystemProfile
{
RdpSettings = new RdpSettings
{
UserName = RdpUser,
Password = RdpPassword,
ExpiryDate = new DateTime(2025, 3, 1)
}
}
}
};
var workerNode = new Role
{
Name = "workernode",
TargetInstanceCount = 5,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
}
};
cluster.Properties.ComputeProfile = new ComputeProfile();
cluster.Properties.ComputeProfile.Roles.Add(headNode);
cluster.Properties.ComputeProfile.Roles.Add(workerNode);
return(cluster);
}
public static void ValidateCluster(ClusterCreateParameters parameters, Cluster cluster, string clustername)
{
ClusterCreateParametersExtended extendedParams = CreateParametersConverter.GetExtendedClusterCreateParameters(clustername, parameters);
ValidateCluster(extendedParams, cluster, clustername);
}
public static ClusterCreateParametersExtended GetIaasClusterSpec(string containerName = DefaultContainer, bool adlStorage = false)
{
StorageInfo storageInfo = adlStorage ? GetDefaultDataLakeStorageInfo() : GetDefaultAzureStorageInfo(containerName);
var cluster = new ClusterCreateParametersExtended
{
Location = Location,
Tags = new Dictionary <string, string>(),
Properties = new ClusterCreateProperties
{
ClusterVersion = "3.6",
OsType = OSType.Linux,
ClusterDefinition = new ClusterDefinition
{
Kind = "Hadoop",
ComponentVersion = new Dictionary <string, string>(),
Configurations = new Dictionary <string, Dictionary <string, string> >()
{
{ "core-site", GetCoreConfigsForStorageInfo(storageInfo) },
{ "gateway", new Dictionary <string, string>
{
{ "restAuthCredential.isEnabled", "true" },
{ "restAuthCredential.username", "admin" },
{ "restAuthCredential.password", "Password1!" }
} }
}
},
Tier = Tier.Standard,
ComputeProfile = new ComputeProfile
{
Roles = new List <Role> {
new Role
{
Name = "headnode",
TargetInstanceCount = 2,
HardwareProfile = new HardwareProfile
{
VmSize = DefaultVmSizes.HeadNode.GetSize("Hadoop")
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
Username = SshUser,
Password = SshPassword
}
}
},
new Role
{
Name = "workernode",
TargetInstanceCount = 3,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
Username = SshUser,
Password = SshPassword
}
}
},
new Role
{
Name = "zookeepernode",
TargetInstanceCount = 3,
HardwareProfile = new HardwareProfile
{
VmSize = DefaultVmSizes.ZookeeperNode.GetSize("Hadoop")
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
Username = SshUser,
Password = SshPassword
}
}
}
}
}
}
};
return(cluster);
}
/// <summary>
/// Creates a new HDInsight cluster with the specified parameters.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.IClusterOperations.
/// </param>
/// <param name='resourceGroupName'>
/// Required. The name of the resource group.
/// </param>
/// <param name='clusterName'>
/// Required. The name of the cluster.
/// </param>
/// <param name='clusterCreateParameters'>
/// Required. The cluster create request.
/// </param>
/// <returns>
/// The GetCluster operation response.
/// </returns>
public static ClusterGetResponse Create(this IClusterOperations operations, string resourceGroupName, string clusterName, ClusterCreateParametersExtended clusterCreateParameters)
{
return(Task.Factory.StartNew((object s) =>
{
return ((IClusterOperations)s).CreateAsync(resourceGroupName, clusterName, clusterCreateParameters);
}
, operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult());
}
/// <summary>
/// Creates a new HDInsight cluster with the specified parameters.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.IClusterOperations.
/// </param>
/// <param name='resourceGroupName'>
/// Required. The name of the resource group.
/// </param>
/// <param name='clusterName'>
/// Required. The name of the cluster.
/// </param>
/// <param name='clusterCreateParameters'>
/// Required. The cluster create request.
/// </param>
/// <returns>
/// The GetCluster operation response.
/// </returns>
public static Task <ClusterGetResponse> CreateAsync(this IClusterOperations operations, string resourceGroupName, string clusterName, ClusterCreateParametersExtended clusterCreateParameters)
{
return(operations.CreateAsync(resourceGroupName, clusterName, clusterCreateParameters, CancellationToken.None));
}
/// <summary>
/// Create cluster create parameters for ADLS Gen1 relevant tests.
/// </summary>
/// <param name="commonData"></param>
/// <param name="createParams">
/// If provided, the method will update the given parameters;
/// Otherwise, a new create parameters will be created.
/// </param>
/// <returns></returns>
public static ClusterCreateParametersExtended PrepareClusterCreateParamsForADLSv1(this CommonTestFixture commonData, ClusterCreateParametersExtended createParams = null)
{
var createParamsForADLSv1 = createParams ?? commonData.PrepareClusterCreateParams();
var configurations = (Dictionary <string, Dictionary <string, string> >)createParamsForADLSv1.Properties.ClusterDefinition.Configurations;
string clusterIdentity = "clusterIdentity";
var clusterIdentityConfig = new Dictionary <string, string>()
{
{ "clusterIdentity.applicationId", commonData.DataLakeClientId },
{ "clusterIdentity.certificate", commonData.CertContent },
{ "clusterIdentity.aadTenantId", "https://login.windows.net/" + commonData.TenantId },
{ "clusterIdentity.resourceUri", "https://datalake.azure.net/" },
{ "clusterIdentity.certificatePassword", commonData.CertPassword }
};
configurations.Add(clusterIdentity, clusterIdentityConfig);
bool isDefault = !createParamsForADLSv1.Properties.StorageProfile.Storageaccounts.Any();
if (isDefault)
{
string coreSite = "core-site";
var coreConfig = new Dictionary <string, string>()
{
{ "fs.defaultFS", "adl://home" },
{ "dfs.adls.home.hostname", commonData.DataLakeStoreAccountName + ".azuredatalakestore.net" },
{ "dfs.adls.home.mountpoint", commonData.DataLakeStoreMountpoint }
};
configurations.Add(coreSite, coreConfig);
}
return(createParamsForADLSv1);
}
internal static void ClassMain(string[] args)
{
// Authentication
var credentials = SdkContext.AzureCredentialsFactory
.FromServicePrincipal(
Configurations.ClientId,
Configurations.ClientSecret,
Configurations.TenantId,
AzureEnvironment.AzureGlobalCloud);
var client = new HDInsightManagementClient(credentials)
{
SubscriptionId = Configurations.SubscriptionId
};
// Parse ADLS Gen2 storage account name from resource id
string adlsGen2AccountName = Configurations.AdlsGen2ResourceId.Split('/').Last();
string clusterName = Configurations.ClusterName;
// Prepare cluster create parameters
var createParamsForADLSv2 = new ClusterCreateParametersExtended
{
Location = Configurations.Location,
Properties = new ClusterCreateProperties
{
ClusterVersion = "3.6",
OsType = OSType.Linux,
Tier = Tier.Standard,
ClusterDefinition = new ClusterDefinition
{
Kind = "Hadoop",
Configurations = new Dictionary <string, Dictionary <string, string> >()
{
{ "gateway", new Dictionary <string, string>
{
{ "restAuthCredential.isEnabled", "true" },
{ "restAuthCredential.username", Configurations.ClusterLoginUserName },
{ "restAuthCredential.password", Configurations.Password }
} }
}
},
ComputeProfile = new ComputeProfile
{
Roles = new List <Role>
{
new Role
{
Name = "headnode",
TargetInstanceCount = 2,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
Username = Configurations.SshUserName,
Password = Configurations.Password
}
}
},
new Role
{
Name = "workernode",
TargetInstanceCount = 3,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
Username = Configurations.SshUserName,
Password = Configurations.Password
}
}
}
}
},
StorageProfile = new StorageProfile
{
Storageaccounts = new List <StorageAccount>
{
new StorageAccount
{
Name = adlsGen2AccountName + Configurations.DfsEndpointSuffix,
IsDefault = true,
FileSystem = Configurations.AdlsGen2FileSystemName.ToLowerInvariant(),
ResourceId = Configurations.AdlsGen2ResourceId,
MsiResourceId = Configurations.ManagedIdentityResourceId
}
}
}
},
Identity = new ClusterIdentity
{
Type = ResourceIdentityType.UserAssigned,
UserAssignedIdentities = new Dictionary <string, ClusterIdentityUserAssignedIdentitiesValue>
{
{ Configurations.ManagedIdentityResourceId, new ClusterIdentityUserAssignedIdentitiesValue() }
}
}
};
Console.WriteLine($"Starting to create HDInsight Hadoop cluster {clusterName} with Azure Data Lake Storage Gen2");
client.Clusters.Create(Configurations.ResourceGroupName, clusterName, createParamsForADLSv2);
Console.WriteLine($"Finished creating HDInsight Hadoop cluster {clusterName} with Azure Data Lake Storage Gen2");
}
private ClusterCreateParametersExtended GetExtendedClusterCreateParameters(
string clusterName, ClusterCreateParameters clusterCreateParameters)
{
var createParamsExtended = new ClusterCreateParametersExtended
{
Location = clusterCreateParameters.Location,
Properties = new ClusterCreateProperties
{
ClusterDefinition = new ClusterDefinition
{
ClusterType = clusterCreateParameters.ClusterType
},
ClusterVersion = clusterCreateParameters.Version,
OperatingSystemType = clusterCreateParameters.OSType,
ClusterTier = clusterCreateParameters.ClusterTier
}
};
var configurations = GetConfigurations(clusterName, clusterCreateParameters);
if (clusterCreateParameters.HiveMetastore != null)
{
var metastoreConfig = GetMetastoreConfig(clusterCreateParameters.HiveMetastore, clusterCreateParameters.OSType, "Hive");
foreach (var configSet in metastoreConfig)
{
if (configurations.ContainsKey(configSet.Key))
{
foreach (var config in configSet.Value)
{
configurations[configSet.Key].Add(config.Key, config.Value);
}
}
else
{
configurations.Add(configSet.Key, configSet.Value);
}
}
}
if (clusterCreateParameters.OozieMetastore != null)
{
var metastoreConfig = GetMetastoreConfig(clusterCreateParameters.OozieMetastore, clusterCreateParameters.OSType, "oozie");
foreach (var configSet in metastoreConfig)
{
if (configurations.ContainsKey(configSet.Key))
{
foreach (var config in configSet.Value)
{
configurations[configSet.Key].Add(config.Key, config.Value);
}
}
else
{
configurations.Add(configSet.Key, configSet.Value);
}
}
}
var serializedConfig = JsonConvert.SerializeObject(configurations);
createParamsExtended.Properties.ClusterDefinition.Configurations = serializedConfig;
var roles = GetRoleCollection(clusterCreateParameters);
createParamsExtended.Properties.ComputeProfile = new ComputeProfile();
foreach (var role in roles)
{
createParamsExtended.Properties.ComputeProfile.Roles.Add(role);
}
return(createParamsExtended);
}
public void ConvertParametersWithMissingNoProvidedPropertiesShouldNotThrow()
{
ClusterCreateParameters createParams = new ClusterCreateParameters();
ClusterCreateParametersExtended extendedParams = CreateParametersConverter.GetExtendedClusterCreateParameters("testCluster", createParams);
}
