public static void ValidateRoles(IList <Role> roleCollection, ClusterCreateParameters createParams)
{
Assert.NotNull(roleCollection);
foreach (Role role in roleCollection)
{
ValidateOsProfile(role.OsProfile, createParams);
ValidateVnet(role.VirtualNetworkProfile, createParams);
Assert.NotNull(role.HardwareProfile);
Assert.True(Enum.TryParse(role.Name, true, out ClusterNodeType nodeType));
Assert.Equal(CreateParametersConverter.GetNodeSize(createParams, nodeType), role.HardwareProfile.VmSize);
if (createParams.ScriptActions != null && createParams.ScriptActions.ContainsKey(nodeType))
{
Assert.Equal(createParams.ScriptActions[nodeType], role.ScriptActions);
}
}
//Validate headnode.
Role headnode = roleCollection.FirstOrDefault(role => role.Name == "headnode");
Assert.NotNull(headnode);
int targetCount = createParams.ClusterType.Equals("Sandbox", StringComparison.OrdinalIgnoreCase) ? 1 : 2;
Assert.Equal(targetCount, headnode.TargetInstanceCount);
//Sandbox clusters only have 1 headnode. Return if Sandbox.
if (createParams.ClusterType.Equals("Sandbox", StringComparison.OrdinalIgnoreCase))
{
Assert.Equal(1, roleCollection.Count);
return;
}
//Validate workernode.
Role workernode = roleCollection.FirstOrDefault(role => role.Name == "workernode");
Assert.NotNull(workernode);
Assert.Equal(createParams.ClusterSizeInNodes, workernode.TargetInstanceCount);
//Validate zookeeper.
Role zookeepernode = roleCollection.FirstOrDefault(role => role.Name == "zookeepernode");
Assert.NotNull(zookeepernode);
Assert.Equal(3, zookeepernode.TargetInstanceCount);
//RServer & MLServices clusters contain an additional edge node. Return if not RServer or not MLServices.
if (!new [] { "RServer", "MLServices" }.Contains(createParams.ClusterType, StringComparer.OrdinalIgnoreCase))
{
Assert.Equal(3, roleCollection.Count);
return;
}
//Validate edgenode.
Role edgenode = roleCollection.FirstOrDefault(role => role.Name == "edgenode");
Assert.NotNull(edgenode);
Assert.Equal(1, edgenode.TargetInstanceCount);
Assert.Equal(4, roleCollection.Count);
}
public static void ValidateSpecConversion(ClusterCreateParameters createParams)
{
//Convert spec.
ClusterCreateParametersExtended extendedParams = CreateParametersConverter.GetExtendedClusterCreateParameters("testCluster", createParams);
//Validate properties.
Assert.Equal(createParams.Tags, extendedParams.Tags);
Assert.Equal(createParams.Location, extendedParams.Location);
Assert.Equal(createParams.ClusterType, extendedParams.Properties.ClusterDefinition.Kind);
Assert.Equal(createParams.ComponentVersion, extendedParams.Properties.ClusterDefinition.ComponentVersion);
Assert.Equal(createParams.ClusterTier, extendedParams.Properties.Tier);
Assert.Equal(createParams.Version, extendedParams.Properties.ClusterVersion);
Assert.Equal(OSType.Linux, extendedParams.Properties.OsType);
Assert.Equal(createParams.SecurityProfile, extendedParams.Properties.SecurityProfile);
ValidateStorageProfile(createParams.DefaultStorageInfo, extendedParams.Properties.StorageProfile);
//Validate configurations.
Dictionary <string, Dictionary <string, string> > configurations = extendedParams.Properties.ClusterDefinition.Configurations as Dictionary <string, Dictionary <string, string> >;
Assert.NotNull(configurations);
ValidateStorageConfigurations(configurations, createParams.DefaultStorageInfo, createParams.AdditionalStorageAccounts);
ValidateClusterCredentialConfigurations(configurations);
ValidateAdlConfigurations(configurations, createParams.Principal);
ValidateHiveMetastoreConfigurations(configurations, createParams.HiveMetastore);
ValidateOozieMetastoreConfigurations(configurations, createParams.OozieMetastore);
//Validate roles.
ValidateRoles(extendedParams.Properties.ComputeProfile.Roles, createParams);
}
public virtual Cluster CreateNewCluster(string resourceGroupName, string clusterName, OSType osType, ClusterCreateParameters parameters)
{
var createParams = CreateParametersConverter.GetExtendedClusterCreateParameters(clusterName, parameters);
createParams.Properties.OsType = osType;
return(HdInsightManagementClient.Clusters.Create(resourceGroupName, clusterName, createParams));
}
public virtual Cluster CreateNewCluster(string resourceGroupName, string clusterName, OSType osType, ClusterCreateParameters parameters,
string minSupportedTlsVersion = default(string), string cloudAadAuthority = default(string),
string cloudDataLakeAudience = default(string), string PublicNetworkAccessType = default(string),
string OutboundOnlyNetworkAccessType = default(string), bool?EnableEncryptionInTransit = default(bool?))
{
var createParams = CreateParametersConverter.GetExtendedClusterCreateParameters(clusterName, parameters);
createParams.Properties.OsType = osType;
createParams.Properties.MinSupportedTlsVersion = minSupportedTlsVersion;
ResetClusterIdentity(createParams, cloudAadAuthority, cloudDataLakeAudience);
if (EnableEncryptionInTransit.HasValue)
{
createParams.Properties.EncryptionInTransitProperties = new EncryptionInTransitProperties()
{
IsEncryptionInTransitEnabled = EnableEncryptionInTransit
};
}
if (!string.IsNullOrEmpty(PublicNetworkAccessType) || !string.IsNullOrEmpty(OutboundOnlyNetworkAccessType))
{
NetworkSettings networkSettings = new NetworkSettings()
{
PublicNetworkAccess = PublicNetworkAccessType,
OutboundOnlyPublicNetworkAccessType = OutboundOnlyNetworkAccessType
};
createParams.Properties.NetworkSettings = networkSettings;
}
return(HdInsightManagementClient.Clusters.Create(resourceGroupName, clusterName, createParams));
}
public virtual Cluster CreateNewCluster(string resourceGroupName, string clusterName, OSType osType, ClusterCreateParameters parameters, string minSupportedTlsVersion = default(string), string cloudAadAuthority = default(string), string cloudDataLakeAudience = default(string))
{
var createParams = CreateParametersConverter.GetExtendedClusterCreateParameters(clusterName, parameters);
createParams.Properties.OsType = osType;
createParams.Properties.MinSupportedTlsVersion = minSupportedTlsVersion;
ResetClusterIdentity(createParams, cloudAadAuthority, cloudDataLakeAudience);
return(HdInsightManagementClient.Clusters.Create(resourceGroupName, clusterName, createParams));
}
public void CanConvertHadoopClusterWithCustomMetastoresToMarchSpecWithConfigurations()
{
ClusterCreateParameters createParams = GetClusterCreateParamsWithMinRequiredValues();
createParams.HiveMetastore = new Metastore("server.database.windows.net", "hiveDb", "username", "password");
createParams.OozieMetastore = new Metastore("server.database.windows.net", "oozieDb", "username", "password");
Dictionary <string, string> testConfig = new Dictionary <string, string>
{
{ "key", "value" }
};
createParams.Configurations.Add(ConfigurationKey.HiveSite, testConfig);
createParams.Configurations.Add(ConfigurationKey.HiveEnv, testConfig);
createParams.Configurations.Add(ConfigurationKey.OozieEnv, testConfig);
createParams.Configurations.Add(ConfigurationKey.OozieSite, testConfig);
ClusterCreateParametersExtended extendedParams = CreateParametersConverter.GetExtendedClusterCreateParameters("testCluster", createParams);
Dictionary <string, Dictionary <string, string> > configurations = extendedParams.Properties.ClusterDefinition.Configurations as Dictionary <string, Dictionary <string, string> >;
Assert.True(configurations.ContainsKey(ConfigurationKey.HiveSite));
Dictionary <string, string> hiveSiteConfig = configurations[ConfigurationKey.HiveSite];
Assert.Equal(5, hiveSiteConfig.Count);
}
public static void ValidateCluster(ClusterCreateParameters parameters, Cluster cluster, string clustername)
{
ClusterCreateParametersExtended extendedParams = CreateParametersConverter.GetExtendedClusterCreateParameters(clustername, parameters);
ValidateCluster(extendedParams, cluster, clustername);
}
public void ConvertParametersWithMissingNoProvidedPropertiesShouldNotThrow()
{
ClusterCreateParameters createParams = new ClusterCreateParameters();
ClusterCreateParametersExtended extendedParams = CreateParametersConverter.GetExtendedClusterCreateParameters("testCluster", createParams);
}
