private async Task ValidateCreateClusterSucceeds(ClusterCreateParametersV2 cluster, Action <ClusterDetails> postCreateValidation)
{
IHDInsightCertificateCredential credentials = IntegrationTestBase.GetValidCredentials();
using (var client = ServiceLocator.Instance.Locate <IHDInsightManagementPocoClientFactory>().Create(credentials, GetAbstractionContext(), false))
{
client.CreateContainer(cluster).Wait();
await client.WaitForClusterInConditionOrError(null, cluster.Name, cluster.Location, TimeSpan.FromMinutes(30), HDInsightClient.DefaultPollingInterval, GetAbstractionContext(), this.CreatingStates);
var task = client.ListContainer(cluster.Name);
task.WaitForResult();
var container = task.Result;
Assert.IsNotNull(container);
if (container.Error.IsNotNull())
{
Assert.Fail("The Container was not expected to return an error but returned ({0}) ({1})", container.Error.HttpCode, container.Error.Message);
}
if (postCreateValidation != null)
{
postCreateValidation(container);
}
await client.DeleteContainer(cluster.Name);
await client.WaitForClusterNullOrError(cluster.Name, cluster.Location, TimeSpan.FromMinutes(10), IntegrationTestBase.GetAbstractionContext().CancellationToken);
Assert.IsNull(container.Error);
Assert.IsNull(client.ListContainer(cluster.Name).WaitForResult());
}
}
private static void ConfigHiveComponent(HiveComponent hive, ClusterCreateParametersV2 inputs)
{
hive.HiveSiteXmlProperties.AddRange(
inputs.HiveConfiguration.ConfigurationCollection.Select(prop => new Property {
Name = prop.Key, Value = prop.Value
}));
if (inputs.HiveConfiguration.AdditionalLibraries != null)
{
hive.AdditionalLibraries = new BlobContainerCredentialBackedResource()
{
AccountDnsName = inputs.HiveConfiguration.AdditionalLibraries.Name,
BlobContainerName = inputs.HiveConfiguration.AdditionalLibraries.Container,
Key = inputs.HiveConfiguration.AdditionalLibraries.Key
};
}
if (inputs.HiveMetastore != null)
{
hive.Metastore = new SqlAzureDatabaseCredentialBackedResource()
{
SqlServerName = inputs.HiveMetastore.Server,
Credentials = new UsernamePasswordCredential()
{
Username = inputs.HiveMetastore.User, Password = inputs.HiveMetastore.Password
},
DatabaseName = inputs.HiveMetastore.Database
};
}
}
public async Task ICanAddAndDeleteRDPUser()
{
IHDInsightSubscriptionCredentials credentials = IntegrationTestBase.GetValidCredentials();
IHDInsightClient client = HDInsightClient.Connect(credentials);
ClusterCreateParametersV2 clusterCreationDetails = GetRandomCluster();
clusterCreationDetails.Version = "1.6";
ClusterDetails clusterDetails = client.CreateCluster(clusterCreationDetails);
// add a rdp user - rdp is OFF by default, hence we enable it first
string userName = "******";
string password = GetRandomValidPassword();
client.EnableRdp(clusterDetails.Name, clusterDetails.Location, userName, password, DateTime.UtcNow.AddHours(1));
ClusterDetails cluster = await client.GetClusterAsync(clusterDetails.Name);
Assert.AreEqual(userName, cluster.RdpUserName, "Rdp user name has not been updated");
// now disable the rdp user
await client.DisableRdpAsync(clusterDetails.Name, clusterDetails.Location);
cluster = await client.GetClusterAsync(clusterDetails.Name);
Assert.IsFalse(String.IsNullOrEmpty(cluster.Name), "Cluster user name is empty, maybe cluster was not created.");
Assert.IsTrue(String.IsNullOrEmpty(cluster.RdpUserName), "Rdp user name has not been cleared");
// delete the cluster
if (!string.Equals(clusterDetails.Name, IntegrationTestBase.TestCredentials.WellKnownCluster.DnsName))
{
client.DeleteCluster(clusterDetails.Name);
}
}
public static ClusterDetails CreateClusterIfNotExists(ClusterCreateParametersV2 parameters)
{
try
{
LOG.InfoFormat("Checking if the HDInsight cluster exists - SubscriptionId: {0}, Name: {1}",
AppConfig.SubscriptionId, parameters.Name);
var cluster = Client.GetCluster(parameters.Name);
if (cluster == null)
{
LOG.InfoFormat("Submitting a new HDInsight cluster create request - - SubscriptionId: {0}, Name: {1}",
AppConfig.SubscriptionId, parameters.Name);
cluster = Client.CreateCluster(parameters);
}
else
{
LOG.InfoFormat("Skipping creation, found an existing HDInsight Cluster with same name. Name: {0}", parameters.Name);
}
if (cluster.State != ClusterState.Operational && cluster.State != ClusterState.Running)
{
throw new ApplicationException(
String.Format("HDInsight Cluster is not in a operational or running state. Name: {0}, State: {1}",
cluster.Name, cluster.StateString));
}
LOG.InfoFormat("HDInsight cluster created. Details:\r\n{0}", ClusterDetailsAsString(cluster));
return(cluster);
}
catch (Exception ex)
{
LOG.Error(
String.Format("Failed to create cluster - SubscriptionId: {0}, Name: {1}", AppConfig.SubscriptionId, parameters.Name),
ex);
throw;
}
}
/// <summary>
/// Converts user supplied cluster create parameters to IaasCluster type as understood by the Resource Provider.
/// </summary>
/// <param name="clusterCreateParameters">User supplied cluster create parameters.</param>
/// <param name="userSubscriptionId">User's subscription Id.</param>
/// <returns>An Instance of IaasCluster</returns>
public static IaasCluster ConvertToIaasCluster(ClusterCreateParametersV2 clusterCreateParameters, string userSubscriptionId)
{
if (clusterCreateParameters == null)
{
throw new ArgumentNullException("clusterCreateParameters");
}
if (String.IsNullOrEmpty(userSubscriptionId))
{
throw new ArgumentNullException("userSubscriptionId");
}
var correlationId = Guid.NewGuid().ToString();
var iaasCluster = new IaasCluster()
{
Id = clusterCreateParameters.Name,
Location = clusterCreateParameters.Location,
ApiVersion = "1.0",
UserSubscriptionId = userSubscriptionId,
UserTags = new Dictionary <string, string>(),
HdiVersion = clusterCreateParameters.Version,
DeploymentDocuments = new Dictionary <string, string>()
{
{ IaasClusterDocumentTypes.EmbeddedAmbariConfigurationDocument, GenerateAmbariConfigurationDocument(clusterCreateParameters) },
{ IaasClusterDocumentTypes.EmbeddedAzureConfigurationDocument, GenerateAzureDocument(clusterCreateParameters) }
}
};
iaasCluster.UserTags.Add("Client", string.Format("HDInsight .Net SDK {0}", Assembly.GetExecutingAssembly().GetName().Version.ToString()));
return(iaasCluster);
}
private static string GenerateAzureDocument(ClusterCreateParametersV2 clusterCreateParameters)
{
if (clusterCreateParameters == null)
{
throw new ArgumentNullException("clusterCreateParameters");
}
string document = VersionToDocumentMapper.GetAzureCsmDocument(!String.IsNullOrEmpty(clusterCreateParameters.SshPassword));
AzureCsmDocumentManager azureCsmDocumentManager = new AzureCsmDocumentManager(document);
// Set basic cluster parameters
azureCsmDocumentManager.SetDnsName(clusterCreateParameters.Name);
azureCsmDocumentManager.SetLocation(clusterCreateParameters.Location);
azureCsmDocumentManager.SetWorkerNodeCount(clusterCreateParameters.ClusterSizeInNodes);
azureCsmDocumentManager.SetHeadNodeVMSize(String.IsNullOrEmpty(clusterCreateParameters.HeadNodeSize) ? NodeVMSize.Large.ToString() : clusterCreateParameters.HeadNodeSize);
azureCsmDocumentManager.SetDataNodeVMSize(String.IsNullOrEmpty(clusterCreateParameters.DataNodeSize) ? NodeVMSize.Large.ToString() : clusterCreateParameters.DataNodeSize);
// Set SSH parameters
if (!String.IsNullOrEmpty(clusterCreateParameters.SshUserName))
{
if (!String.IsNullOrEmpty(clusterCreateParameters.SshPassword))
{
azureCsmDocumentManager.SetSshProfile(clusterCreateParameters.SshUserName, clusterCreateParameters.SshPassword);
}
else
{
var x509cert = GetOpenSshCertificate(clusterCreateParameters.SshPublicKey, String.Format("CN={0}", clusterCreateParameters.Name));
azureCsmDocumentManager.SetSshProfile(clusterCreateParameters.SshUserName, x509cert);
}
}
return(azureCsmDocumentManager.Document);
}
public ClusterDetails CreateCluster(ClusterCreateParametersV2 cluster)
{
Task <ClusterDetails> createTask = this.CreateClusterAsync(cluster);
createTask.Wait();
return(createTask.Result);
}
public static ClusterDetails CreateClusterIfNotExists(ClusterType clusterType)
{
var name = AppConfig.AzureResourceName + clusterType.ToString().ToLowerInvariant();
LOG.Info(String.Format("Creating Cluster: Name = {0}, Type = {1}", name, clusterType));
var parameters = new ClusterCreateParametersV2()
{
Name = name,
ClusterSizeInNodes = HDInsightClusterSize,
Location = AppConfig.AzureResourceLocation,
ClusterType = clusterType,
HeadNodeSize = "Large",
DataNodeSize = "Large",
ZookeeperNodeSize = "Medium",
UserName = AppConfig.AzureResourceUsername,
Password = AppConfig.AzureResourcePassword,
DefaultStorageAccountName = AzureStorageHelper.Name,
DefaultStorageAccountKey = AzureStorageHelper.PrimaryKey,
DefaultStorageContainer = AppConfig.AzureResourceName,
Version = "3.2",
//VirtualNetworkId = VirtualNetworkHelper.VNetId,
//SubnetName = VirtualNetworkHelper.SubnetName,
};
return(CreateClusterIfNotExists(parameters));
}
public override async Task EndProcessing()
{
IHDInsightClient client = this.GetClient(IgnoreSslErrors);
client.ClusterProvisioning += this.ClientOnClusterProvisioning;
ClusterCreateParametersV2 createClusterRequest = this.GetClusterCreateParameters();
var cluster = await client.CreateClusterAsync(createClusterRequest);
this.Output.Add(new AzureHDInsightCluster(cluster));
}
private static WabStorageAccountConfiguration GetStorageAccountForScript(ScriptAction sa, ClusterCreateParametersV2 details)
{
var accts = new List<WabStorageAccountConfiguration>();
accts.Add(new WabStorageAccountConfiguration(
details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer));
accts.AddRange(details.AdditionalStorageAccounts);
// Tests whether the host for the script is in the list of provided storage accounts.
var storage = (from acct in accts
where GetFullyQualifiedStorageAccountName(acct.Name).Equals(
sa.Uri.Host, StringComparison.OrdinalIgnoreCase)
select acct).FirstOrDefault();
return storage;
}
private async Task ValidateCreateClusterFailsWithError(ClusterCreateParametersV2 cluster)
{
IHDInsightCertificateCredential credentials = IntegrationTestBase.GetValidCredentials();
var client = ServiceLocator.Instance.Locate <IHDInsightManagementPocoClientFactory>().Create(credentials, GetAbstractionContext(), false);
await client.CreateContainer(cluster);
await client.WaitForClusterNotNull(cluster.Name, cluster.Location, TimeSpan.FromMinutes(10), GetAbstractionContext().CancellationToken);
var result = await client.ListContainer(cluster.Name);
Assert.IsNotNull(result);
Assert.IsNotNull(result.Error);
await client.DeleteContainer(cluster.Name);
await client.WaitForClusterNull(cluster.Name, cluster.Location, TimeSpan.FromMinutes(10), GetAbstractionContext().CancellationToken);
}
private static bool CreateHasNewVMSizesSpecified(ClusterCreateParametersV2 clusterCreateParameters)
{
return(new[]
{
clusterCreateParameters.HeadNodeSize,
clusterCreateParameters.DataNodeSize,
clusterCreateParameters.ZookeeperNodeSize
}
.Except(
new[]
{
"ExtraLarge",
"Large",
"Medium",
"Small",
"ExtraSmall"
}, StringComparer.OrdinalIgnoreCase).Any(ns => ns.IsNotNullOrEmpty()));
}
private static string GenerateAmbariConfigurationDocument(ClusterCreateParametersV2 clusterCreateParameters)
{
if (clusterCreateParameters == null)
{
throw new ArgumentNullException("clusterCreateParameters");
}
string document = VersionToDocumentMapper.GetAmbariConfigurationDocument(clusterCreateParameters.Version);
AmbariConfigurationDocumentManager ambariConfigurationManager = new AmbariConfigurationDocumentManager(document);
// Set password
ambariConfigurationManager.SetPassword(clusterCreateParameters.Password);
// Set default storage account information
ambariConfigurationManager.RemoveStorageAccountEntries();
ambariConfigurationManager.SetDefaultStorageAccount(clusterCreateParameters.DefaultStorageContainer, clusterCreateParameters.DefaultStorageAccountName, clusterCreateParameters.DefaultStorageAccountKey);
// Set additional storage accounts
foreach (var storageAccount in clusterCreateParameters.AdditionalStorageAccounts)
{
ambariConfigurationManager.SetAdditionalStorageAccount(storageAccount.Name, storageAccount.Key);
}
// Set custom metastores
if (clusterCreateParameters.HiveMetastore != null)
{
ambariConfigurationManager.SetCustomHiveMetastore(clusterCreateParameters.HiveMetastore);
}
if (clusterCreateParameters.OozieMetastore != null)
{
ambariConfigurationManager.SetCustomOozieMetastore(clusterCreateParameters.OozieMetastore);
}
// Set user specified Hadoop configurations
ambariConfigurationManager.SetCustomConfigurations(AmbariConfigurationDocumentManager.CoreConfigurationKeyName, clusterCreateParameters.CoreConfiguration);
ambariConfigurationManager.SetCustomConfigurations(AmbariConfigurationDocumentManager.HdfsConfigurationKeyName, clusterCreateParameters.HdfsConfiguration);
ambariConfigurationManager.SetCustomConfigurations(AmbariConfigurationDocumentManager.YarnConfigurationKeyName, clusterCreateParameters.YarnConfiguration);
ambariConfigurationManager.SetCustomConfigurations(AmbariConfigurationDocumentManager.HiveConfigurationKeyName, clusterCreateParameters.HiveConfiguration.ConfigurationCollection);
ambariConfigurationManager.SetCustomConfigurations(AmbariConfigurationDocumentManager.OozieConfigurationKeyName, clusterCreateParameters.OozieConfiguration.ConfigurationCollection);
ambariConfigurationManager.SetCustomConfigurations(AmbariConfigurationDocumentManager.MapredConfigurationKeyName, clusterCreateParameters.MapReduceConfiguration.ConfigurationCollection);
return(ambariConfigurationManager.Document);
}
public Task<ClusterDetails> CreateClusterAsync(ClusterCreateParametersV2 clusterCreateParameters)
{
this.LogMessage("Creating cluster '{0}' in location {1}", clusterCreateParameters.Name, clusterCreateParameters.Location);
LastCreateRequest = clusterCreateParameters;
var clusterDetails = new ClusterDetails();
clusterDetails.Name = clusterCreateParameters.Name;
clusterDetails.HttpPassword = clusterCreateParameters.Password;
clusterDetails.HttpUserName = clusterCreateParameters.UserName;
clusterDetails.Version = clusterCreateParameters.Version;
clusterDetails.Location = clusterCreateParameters.Location;
clusterDetails.State = ClusterState.Running;
clusterDetails.AdditionalStorageAccounts = clusterCreateParameters.AdditionalStorageAccounts;
clusterDetails.DefaultStorageAccount = new WabStorageAccountConfiguration(
clusterCreateParameters.DefaultStorageAccountName,
clusterCreateParameters.DefaultStorageAccountKey,
clusterCreateParameters.DefaultStorageContainer);
Clusters.Add(new SimulatorClusterContainer { Cluster = clusterDetails });
return TaskEx2.FromResult(clusterDetails);
}
/// <summary>
/// Validates, appends the FQDN suffix if required to storage accounts and creates the default cluster specified in <paramref name="details"/>.
/// </summary>
/// <param name="details">The details.</param>
public static void ValidateAndResolveAsvAccountsAndPrep(ClusterCreateParametersV2 details)
{
var defaultStorageAccount = new WabStorageAccountConfiguration(
details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer);
// Flattens all the configurations into a single list for more uniform validation
var asvList = ResolveStorageAccounts(details.AdditionalStorageAccounts).ToList();
asvList.Add(ResolveStorageAccount(defaultStorageAccount));
// Basic validation on the ASV configurations
if (string.IsNullOrEmpty(details.DefaultStorageContainer))
{
throw new InvalidOperationException("Invalid Container. Default Storage Account Container cannot be null or empty");
}
if (asvList.Any(asv => string.IsNullOrEmpty(asv.Name) || string.IsNullOrEmpty(asv.Key)))
{
throw new InvalidOperationException("Invalid Azure Configuration. Credentials cannot be null or empty");
}
if (asvList.GroupBy(asv => asv.Name).Count(group => group.Count() > 1) > 0)
{
throw new InvalidOperationException("Invalid Azure Storage credential. Duplicated values detected");
}
// Validates that we can establish the connection to the ASV Names and the default container
var client = ServiceLocator.Instance.Locate <IAsvValidatorClientFactory>().Create();
asvList.ForEach(asv => client.ValidateAccount(asv.Name, asv.Key).Wait());
var resolvedAccounts = ResolveStorageAccounts(details.AdditionalStorageAccounts);
details.AdditionalStorageAccounts.Clear();
foreach (var resolvedAccount in resolvedAccounts)
{
details.AdditionalStorageAccounts.Add(resolvedAccount);
}
var resolvedDefaultStorageAccount = ResolveStorageAccount(defaultStorageAccount);
details.DefaultStorageAccountName = resolvedDefaultStorageAccount.Name;
client.CreateContainerIfNotExists(details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer).Wait();
}
/// <summary>
/// Validates, appends the FQDN suffix if required to storage accounts and creates the default cluster specified in <paramref name="details"/>.
/// </summary>
/// <param name="details">The details.</param>
public static void ValidateAndResolveAsvAccountsAndPrep(ClusterCreateParametersV2 details)
{
var defaultStorageAccount = new WabStorageAccountConfiguration(
details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer);
// Flattens all the configurations into a single list for more uniform validation
var asvList = ResolveStorageAccounts(details.AdditionalStorageAccounts).ToList();
asvList.Add(ResolveStorageAccount(defaultStorageAccount));
// Basic validation on the ASV configurations
if (string.IsNullOrEmpty(details.DefaultStorageContainer))
{
throw new InvalidOperationException("Invalid Container. Default Storage Account Container cannot be null or empty");
}
if (asvList.Any(asv => string.IsNullOrEmpty(asv.Name) || string.IsNullOrEmpty(asv.Key)))
{
throw new InvalidOperationException("Invalid Azure Configuration. Credentials cannot be null or empty");
}
if (asvList.GroupBy(asv => asv.Name).Count(group => group.Count() > 1) > 0)
{
throw new InvalidOperationException("Invalid Azure Storage credential. Duplicated values detected");
}
// Validates that we can establish the connection to the ASV Names and the default container
var client = ServiceLocator.Instance.Locate<IAsvValidatorClientFactory>().Create();
asvList.ForEach(asv => client.ValidateAccount(asv.Name, asv.Key).Wait());
var resolvedAccounts = ResolveStorageAccounts(details.AdditionalStorageAccounts);
details.AdditionalStorageAccounts.Clear();
foreach (var resolvedAccount in resolvedAccounts)
{
details.AdditionalStorageAccounts.Add(resolvedAccount);
}
var resolvedDefaultStorageAccount = ResolveStorageAccount(defaultStorageAccount);
details.DefaultStorageAccountName = resolvedDefaultStorageAccount.Name;
client.CreateContainerIfNotExists(details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer).Wait();
}
public static void ValidateAndResolveConfigActionEndpointUris(ClusterCreateParametersV2 details)
{
if (details == null || details.ConfigActions == null)
{
return;
}
foreach (ConfigAction ca in details.ConfigActions)
{
ScriptAction sa = ca as ScriptAction;
if (sa == null)
{
continue;
}
// Basic validation on the script action URI.
if (sa.Uri == null || string.IsNullOrEmpty(sa.Uri.AbsoluteUri))
{
throw new InvalidOperationException("Invalid Container. Script action URI cannot be null or empty");
}
var storageAccount = GetStorageAccountForScript(sa, details);
if (storageAccount != null)
{
// Check if the URI is in one of the provided storage accounts and whether it is reachable.
ValidateAndResolveWasbScriptActionEndpointUri(sa.Uri, storageAccount).Wait();
}
else
{
// Check if the URI is publicly reachable in Http format.
ValidateAndResolveHttpScriptActionEndpointUri(sa.Uri).Wait();
}
}
}
public static void ValidateAndResolveConfigActionEndpointUris(ClusterCreateParametersV2 details)
{
if (details == null || details.ConfigActions == null)
{
return;
}
foreach (ConfigAction ca in details.ConfigActions)
{
ScriptAction sa = ca as ScriptAction;
if (sa == null)
{
continue;
}
// Basic validation on the script action URI.
if (sa.Uri == null || string.IsNullOrEmpty(sa.Uri.AbsoluteUri))
{
throw new InvalidOperationException("Invalid Container. Script action URI cannot be null or empty");
}
var storageAccount = GetStorageAccountForScript(sa, details);
if (storageAccount != null)
{
// Check if the URI is in one of the provided storage accounts and whether it is reachable.
ValidateAndResolveWasbScriptActionEndpointUri(sa.Uri, storageAccount).Wait();
}
else
{
// Check if the URI is publicly reachable in Http format.
ValidateAndResolveHttpScriptActionEndpointUri(sa.Uri).Wait();
}
}
}
private void fixDefaultExpectedZookeeperSize(ClusterCreateParametersV2 expected)
{
if (expected.ClusterType == ClusterType.HBase)
{
expected.ZookeeperNodeSize = "Medium";
}
else
{
expected.ZookeeperNodeSize = "Small";
}
}
internal ClusterCreateParametersV2 GetClusterCreateParameters()
{
var createClusterRequest = new ClusterCreateParametersV2();
createClusterRequest.Name = this.Name;
createClusterRequest.Version = this.Version;
createClusterRequest.Location = this.Location;
createClusterRequest.CoreConfiguration.AddRange(this.CoreConfiguration);
createClusterRequest.YarnConfiguration.AddRange(this.YarnConfiguration);
createClusterRequest.HdfsConfiguration.AddRange(this.HdfsConfiguration);
createClusterRequest.MapReduceConfiguration.ConfigurationCollection.AddRange(this.MapReduceConfiguration.ConfigurationCollection);
createClusterRequest.MapReduceConfiguration.CapacitySchedulerConfigurationCollection.AddRange(
this.MapReduceConfiguration.CapacitySchedulerConfigurationCollection);
createClusterRequest.HiveConfiguration.AdditionalLibraries = this.HiveConfiguration.AdditionalLibraries;
createClusterRequest.HiveConfiguration.ConfigurationCollection.AddRange(this.HiveConfiguration.ConfigurationCollection);
createClusterRequest.OozieConfiguration.ConfigurationCollection.AddRange(this.OozieConfiguration.ConfigurationCollection);
createClusterRequest.OozieConfiguration.AdditionalSharedLibraries = this.OozieConfiguration.AdditionalSharedLibraries;
createClusterRequest.OozieConfiguration.AdditionalActionExecutorLibraries = this.OozieConfiguration.AdditionalActionExecutorLibraries;
createClusterRequest.StormConfiguration.AddRange(this.StormConfiguration);
createClusterRequest.SparkConfiguration.AddRange(this.SparkConfiguration);
createClusterRequest.HBaseConfiguration.AdditionalLibraries = this.HBaseConfiguration.AdditionalLibraries;
createClusterRequest.HBaseConfiguration.ConfigurationCollection.AddRange(this.HBaseConfiguration.ConfigurationCollection);
createClusterRequest.DefaultStorageAccountName = this.DefaultStorageAccountName;
createClusterRequest.DefaultStorageAccountKey = this.DefaultStorageAccountKey;
createClusterRequest.DefaultStorageContainer = this.DefaultStorageContainerName;
createClusterRequest.UserName = this.Credential.UserName;
createClusterRequest.Password = this.Credential.GetCleartextPassword();
createClusterRequest.ClusterSizeInNodes = this.ClusterSizeInNodes;
createClusterRequest.ClusterType = this.ClusterType;
if (!string.IsNullOrEmpty(this.VirtualNetworkId))
{
createClusterRequest.VirtualNetworkId = this.VirtualNetworkId;
}
if (!string.IsNullOrEmpty(this.SubnetName))
{
createClusterRequest.SubnetName = this.SubnetName;
}
createClusterRequest.AdditionalStorageAccounts.AddRange(
this.AdditionalStorageAccounts.Select(act => new WabStorageAccountConfiguration(act.StorageAccountName, act.StorageAccountKey)));
createClusterRequest.ConfigActions.AddRange(this.ConfigActions.Select(ca => ca.ToSDKConfigAction()));
if (this.HiveMetastore.IsNotNull())
{
createClusterRequest.HiveMetastore = new Metastore(
this.HiveMetastore.SqlAzureServerName,
this.HiveMetastore.DatabaseName,
this.HiveMetastore.Credential.UserName,
this.HiveMetastore.Credential.GetCleartextPassword());
}
if (this.OozieMetastore.IsNotNull())
{
createClusterRequest.OozieMetastore = new Metastore(
this.OozieMetastore.SqlAzureServerName,
this.OozieMetastore.DatabaseName,
this.OozieMetastore.Credential.UserName,
this.OozieMetastore.Credential.GetCleartextPassword());
}
if (!string.IsNullOrEmpty(this.HeadNodeSize) && !this.HeadNodeSize.Equals("Default", StringComparison.OrdinalIgnoreCase))
{
createClusterRequest.HeadNodeSize = this.HeadNodeSize;
}
if (!string.IsNullOrEmpty(this.DataNodeSize))
{
createClusterRequest.DataNodeSize = this.DataNodeSize;
}
if (!string.IsNullOrEmpty(this.ZookeeperNodeSize))
{
createClusterRequest.ZookeeperNodeSize = this.ZookeeperNodeSize;
}
if (this.RdpCredential.IsNotNull())
{
createClusterRequest.RdpUsername = this.RdpCredential.UserName;
createClusterRequest.RdpPassword = this.RdpCredential.GetCleartextPassword();
}
if (RdpAccessExpiry.IsNotNull())
{
createClusterRequest.RdpAccessExpiry = this.RdpAccessExpiry;
}
// Set IaaS specific parameters
createClusterRequest.OSType = this.OSType;
if (SshCredential != null)
{
createClusterRequest.SshUserName = this.SshCredential.UserName;
createClusterRequest.SshPassword = this.SshCredential.GetCleartextPassword();
}
if (!string.IsNullOrEmpty(SshPublicKey))
{
createClusterRequest.SshPublicKey = this.SshPublicKey;
}
return(createClusterRequest);
}
/// <summary>
/// Creates the wire contract request from user's cluster create parameters.
/// </summary>
/// <param name="cluster">The cluster.</param>
/// <returns>An Instance of cluster create parameters.</returns>
public static ClusterCreateParameters CreateWireClusterCreateParametersFromUserType(ClusterCreateParametersV2 cluster)
{
if (cluster == null)
{
throw new ArgumentNullException("cluster");
}
ClusterCreateParameters ccp = null;
if (cluster.Version.Equals("default", StringComparison.OrdinalIgnoreCase) || new Version(cluster.Version).Major >= 3)
{
switch (cluster.ClusterType)
{
case ClusterType.HBase:
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndHBaseTemplate(cluster);
break;
case ClusterType.Storm:
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndStormTemplate(cluster);
break;
case ClusterType.Hadoop:
ccp = HDInsightClusterRequestGenerator.Create3XClusterFromMapReduceTemplate(cluster);
break;
case ClusterType.Spark:
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndSparkTemplate(cluster);
break;
default:
throw new InvalidDataException(
string.Format(
CultureInfo.InvariantCulture,
"Invalid cluster type '{0}' specified for cluster '{1}'",
cluster.ClusterType,
cluster.Name));
}
}
else
{
if (cluster.ClusterType != ClusterType.Hadoop)
{
throw new InvalidDataException(string.Format(CultureInfo.InvariantCulture, "Invalid cluster type '{0}' specified for cluster '{1}'", cluster.ClusterType, cluster.Name));
}
ccp = new Version(cluster.Version).Major > 1 ? HDInsightClusterRequestGenerator.Create2XClusterForMapReduceTemplate(cluster)
: HDInsightClusterRequestGenerator.Create1XClusterForMapReduceTemplate(cluster);
}
return(ccp);
}
internal static void ValidateClusterConfiguration(ClusterDetails testCluster, ClusterCreateParametersV2 cluster)
{
var remoteCreds = new BasicAuthCredential()
{
Server = GatewayUriResolver.GetGatewayUri(new Uri(testCluster.ConnectionUrl).Host),
UserName = testCluster.HttpUserName,
Password = testCluster.HttpPassword
};
var configurationAccessor =
ServiceLocator.Instance.Locate <IAzureHDInsightClusterConfigurationAccessorFactory>().Create(remoteCreds);
var coreConfiguration = configurationAccessor.GetCoreServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.CoreConfiguration, coreConfiguration);
var hdfsConfiguration = configurationAccessor.GetHdfsServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.HdfsConfiguration, hdfsConfiguration);
var mapReduceConfiguration = configurationAccessor.GetMapReduceServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.MapReduceConfiguration.ConfigurationCollection, mapReduceConfiguration);
var hiveConfiguration = configurationAccessor.GetHiveServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.HiveConfiguration.ConfigurationCollection, hiveConfiguration);
var oozieConfiguration = configurationAccessor.GetOozieServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.OozieConfiguration.ConfigurationCollection, oozieConfiguration);
}
public void ICanCreateAClusterUsingPowerShellAndConfig_New_Set_Add_Hive_Oozie_And_CoreConfig()
{
AzureTestCredentials creds = GetCredentials(TestCredentialsNames.Default);
string dnsName = this.GetRandomClusterName();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.EndProcessing();
int expected = getCommand.Output.Count();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig)
.WithParameter(CmdletConstants.ClusterSizeInNodes, 3)
.AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage)
.WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].DefaultStorageAccount.Name)
.WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].DefaultStorageAccount.Key)
.WithParameter(CmdletConstants.StorageContainerName, creds.Environments[0].DefaultStorageAccount.Container)
.AddCommand(CmdletConstants.AddAzureHDInsightStorage)
.WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].AdditionalStorageAccounts[0].Name)
.WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].AdditionalStorageAccounts[0].Key)
.AddCommand(CmdletConstants.AddAzureHDInsightMetastore)
.WithParameter(CmdletConstants.SqlAzureServerName, creds.Environments[0].HiveStores[0].SqlServer)
.WithParameter(CmdletConstants.DatabaseName, creds.Environments[0].HiveStores[0].Database)
.WithParameter(CmdletConstants.Credential, GetPSCredential(creds.AzureUserName, creds.AzurePassword))
.WithParameter(CmdletConstants.MetastoreType, AzureHDInsightMetastoreType.HiveMetastore)
.AddCommand(CmdletConstants.AddAzureHDInsightMetastore)
.WithParameter(CmdletConstants.SqlAzureServerName, creds.Environments[0].OozieStores[0].SqlServer)
.WithParameter(CmdletConstants.DatabaseName, creds.Environments[0].OozieStores[0].Database)
.WithParameter(CmdletConstants.Credential, GetPSCredential(creds.AzureUserName, creds.AzurePassword))
.WithParameter(CmdletConstants.MetastoreType, AzureHDInsightMetastoreType.OozieMetastore)
.AddCommand(CmdletConstants.NewAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, dnsName)
.WithParameter(CmdletConstants.Location, CmdletConstants.EastUs)
.WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword()))
.Invoke();
ClusterCreateParametersV2 request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest;
Assert.IsNotNull(request.HiveMetastore);
Assert.IsNotNull(request.OozieMetastore);
Assert.AreEqual(1, results.Results.Count);
Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name);
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.Name = dnsName;
getCommand.EndProcessing();
Assert.AreEqual(1, getCommand.Output.Count);
Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name);
results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, dnsName)
.Invoke();
Assert.AreEqual(0, results.Results.Count);
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.EndProcessing();
Assert.AreEqual(expected, getCommand.Output.Count);
}
}
public async Task CreateContainer(ClusterCreateParametersV2 details)
{
this.LogMessage("Create Cluster Requested", Severity.Informational, Verbosity.Diagnostic);
// Validates that the AzureStorage Configurations are valid and optionally append FQDN suffix to the storage account name
AsvValidationHelper.ValidateAndResolveAsvAccountsAndPrep(details);
// Validates that the config actions' Uris are downloadable.
UriEndpointValidator.ValidateAndResolveConfigActionEndpointUris(details);
var overrideHandlers =
ServiceLocator.Instance.Locate <IHDInsightClusterOverrideManager>()
.GetHandlers(this.credentials, this.Context, this.ignoreSslErrors);
var rdfeCapabilitiesClient =
ServiceLocator.Instance.Locate <IRdfeServiceRestClientFactory>()
.Create(this.credentials, this.Context, this.ignoreSslErrors);
var capabilities = await rdfeCapabilitiesClient.GetResourceProviderProperties();
if (!this.HasClusterCreateCapability(capabilities))
{
throw new InvalidOperationException(string.Format(
"Your subscription cannot create clusters, please contact Support"));
}
// For container resource type, config actions should never be enabled
if (details.ConfigActions != null && details.ConfigActions.Count > 0)
{
throw new InvalidOperationException(string.Format(
"Your subscription cannot create customized clusters, please contact Support"));
}
if (!new[] { "ExtraLarge", "Large" }.Contains(details.HeadNodeSize, StringComparer.OrdinalIgnoreCase) ||
!string.Equals(details.DataNodeSize, "Large", StringComparison.OrdinalIgnoreCase) ||
details.ZookeeperNodeSize.IsNotNullOrEmpty())
{
throw new InvalidOperationException(string.Format(
"Illegal node size specification for container resource. Headnode: [{0}], Datanode: [{1}], Zookeeper: [{2}]",
details.HeadNodeSize, details.DataNodeSize, details.ZookeeperNodeSize));
}
// Validates the region for the cluster creation
var locationClient = ServiceLocator.Instance.Locate <ILocationFinderClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors);
var availableLocations = locationClient.ListAvailableLocations(capabilities);
if (!availableLocations.Contains(details.Location, StringComparer.OrdinalIgnoreCase))
{
throw new InvalidOperationException(string.Format(
"Cannot create a cluster in '{0}'. Available Locations for your subscription are: {1}",
details.Location,
string.Join(",", availableLocations)));
}
// Validates whether the subscription\location needs to be initialized
var registrationClient = ServiceLocator.Instance.Locate <ISubscriptionRegistrationClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors);
await registrationClient.RegisterSubscription();
if (!await registrationClient.ValidateSubscriptionLocation(details.Location))
{
await registrationClient.RegisterSubscriptionLocation(details.Location);
}
// Creates the cluster
var client = ServiceLocator.Instance.Locate <IHDInsightManagementRestClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors);
if (details.ClusterType == ClusterType.HBase || details.ClusterType == ClusterType.Storm || details.ClusterType == ClusterType.Spark)
{
string payload = overrideHandlers.PayloadConverter.SerializeClusterCreateRequestV3(details);
await client.CreateContainer(details.Name, details.Location, payload, 3);
}
else
{
if (!details.VirtualNetworkId.IsNullOrEmpty() || !details.SubnetName.IsNullOrEmpty())
{
throw new InvalidOperationException("Create Hadoop clusters within a virtual network is not permitted.");
}
string payload = overrideHandlers.PayloadConverter.SerializeClusterCreateRequest(details);
await client.CreateContainer(details.Name, details.Location, payload);
}
}
private async Task ValidateCreateClusterSucceeds(ClusterCreateParametersV2 cluster)
{
await ValidateCreateClusterSucceeds(cluster, null);
}
private static WabStorageAccountConfiguration GetStorageAccountForScript(ScriptAction sa, ClusterCreateParametersV2 details)
{
var accts = new List <WabStorageAccountConfiguration>();
accts.Add(new WabStorageAccountConfiguration(
details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer));
accts.AddRange(details.AdditionalStorageAccounts);
// Tests whether the host for the script is in the list of provided storage accounts.
var storage = (from acct in accts
where GetFullyQualifiedStorageAccountName(acct.Name).Equals(
sa.Uri.Host, StringComparison.OrdinalIgnoreCase)
select acct).FirstOrDefault();
return(storage);
}
public Task CreateContainer(ClusterCreateParametersV2 details)
{
this.LastCreateRequest = details;
return(underlying.CreateContainer(details));
}
public ClusterDetails CreateCluster(ClusterCreateParametersV2 cluster, TimeSpan timeout)
{
return(this.CreateCluster(cluster));
}
public void ICanCreateAClusterUsingPowerShellAndConfig_New_Set_Add_ScriptAction()
{
AzureTestCredentials creds = GetCredentials(TestCredentialsNames.Default);
string dnsName = this.GetRandomClusterName();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.EndProcessing();
int expected = getCommand.Output.Count();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig)
.WithParameter(CmdletConstants.ClusterSizeInNodes, 3)
.AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage)
.WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].DefaultStorageAccount.Name)
.WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].DefaultStorageAccount.Key)
.WithParameter(CmdletConstants.StorageContainerName, creds.Environments[0].DefaultStorageAccount.Container)
.AddCommand(CmdletConstants.AddAzureHDInsightStorage)
.WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].AdditionalStorageAccounts[0].Name)
.WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].AdditionalStorageAccounts[0].Key)
.AddCommand(CmdletConstants.AddAzureHDInsightScriptAction)
.WithParameter(CmdletConstants.ConfigActionName, "test1")
.WithParameter(CmdletConstants.ConfigActionClusterRoleCollection, "HeadNode")
.WithParameter(CmdletConstants.ScriptActionUri, "http://test1.com")
.WithParameter(CmdletConstants.ScriptActionParameters, "test1parameters")
.AddCommand(CmdletConstants.AddAzureHDInsightScriptAction)
.WithParameter(CmdletConstants.ConfigActionName, "test2")
.WithParameter(CmdletConstants.ConfigActionClusterRoleCollection, "HeadNode")
.WithParameter(CmdletConstants.ScriptActionUri, "http://test2.com")
.WithParameter(CmdletConstants.ScriptActionParameters, "test2parameters")
.AddCommand(CmdletConstants.NewAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, dnsName)
.WithParameter(CmdletConstants.Location, CmdletConstants.EastUs)
.WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword()))
.Invoke();
ClusterCreateParametersV2 request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest;
Assert.IsTrue(request.ConfigActions != null && request.ConfigActions.Count == 2);
Assert.IsTrue(
request.ConfigActions.ElementAt(0).Name == "test1" &&
request.ConfigActions.ElementAt(1).Name == "test2");
Assert.IsTrue(
request.ConfigActions.ElementAt(0).ClusterRoleCollection.Count == 1 &&
request.ConfigActions.ElementAt(1).ClusterRoleCollection.Count == 1);
Assert.AreEqual(1, results.Results.Count);
Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name);
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.Name = dnsName;
getCommand.EndProcessing();
Assert.AreEqual(1, getCommand.Output.Count);
Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name);
results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, dnsName)
.Invoke();
Assert.AreEqual(0, results.Results.Count);
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.EndProcessing();
Assert.AreEqual(expected, getCommand.Output.Count);
}
}
/// <summary>
/// Creates the wire contract request from user's cluster create parameters.
/// </summary>
/// <param name="cluster">The cluster.</param>
/// <returns>An Instance of cluster create parameters.</returns>
public static ClusterCreateParameters CreateWireClusterCreateParametersFromUserType(ClusterCreateParametersV2 cluster)
{
if (cluster == null)
{
throw new ArgumentNullException("cluster");
}
ClusterCreateParameters ccp = null;
if (cluster.Version.Equals("default", StringComparison.OrdinalIgnoreCase) || new Version(ClusterVersionUtils.TryGetVersionNumber(cluster.Version)).Major >= 3)
{
switch (cluster.ClusterType)
{
case ClusterType.HBase:
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndHBaseTemplate(cluster);
break;
case ClusterType.Storm:
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndStormTemplate(cluster);
break;
case ClusterType.Hadoop:
ccp = HDInsightClusterRequestGenerator.Create3XClusterFromMapReduceTemplate(cluster);
break;
case ClusterType.Spark:
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndSparkTemplate(cluster);
break;
default:
throw new InvalidDataException(
string.Format(
CultureInfo.InvariantCulture,
"Invalid cluster type '{0}' specified for cluster '{1}'",
cluster.ClusterType,
cluster.Name));
}
}
else
{
if (cluster.ClusterType != ClusterType.Hadoop)
{
throw new InvalidDataException(string.Format(CultureInfo.InvariantCulture, "Invalid cluster type '{0}' specified for cluster '{1}'", cluster.ClusterType, cluster.Name));
}
ccp = new Version(cluster.Version).Major > 1 ? HDInsightClusterRequestGenerator.Create2XClusterForMapReduceTemplate(cluster)
: HDInsightClusterRequestGenerator.Create1XClusterForMapReduceTemplate(cluster);
}
return ccp;
}
private static void ConfigHiveComponent(HiveComponent hive, ClusterCreateParametersV2 inputs)
{
hive.HiveSiteXmlProperties.AddRange(
inputs.HiveConfiguration.ConfigurationCollection.Select(prop => new Property { Name = prop.Key, Value = prop.Value }));
if (inputs.HiveConfiguration.AdditionalLibraries != null)
{
hive.AdditionalLibraries = new BlobContainerCredentialBackedResource()
{
AccountDnsName = inputs.HiveConfiguration.AdditionalLibraries.Name,
BlobContainerName = inputs.HiveConfiguration.AdditionalLibraries.Container,
Key = inputs.HiveConfiguration.AdditionalLibraries.Key
};
}
if (inputs.HiveMetastore != null)
{
hive.Metastore = new SqlAzureDatabaseCredentialBackedResource()
{
SqlServerName = inputs.HiveMetastore.Server,
Credentials = new UsernamePasswordCredential() { Username = inputs.HiveMetastore.User, Password = inputs.HiveMetastore.Password },
DatabaseName = inputs.HiveMetastore.Database
};
}
}
