private async Task ValidateCreateClusterSucceeds(ClusterCreateParametersV2 cluster, Action <ClusterDetails> postCreateValidation) { IHDInsightCertificateCredential credentials = IntegrationTestBase.GetValidCredentials(); using (var client = ServiceLocator.Instance.Locate <IHDInsightManagementPocoClientFactory>().Create(credentials, GetAbstractionContext(), false)) { client.CreateContainer(cluster).Wait(); await client.WaitForClusterInConditionOrError(null, cluster.Name, cluster.Location, TimeSpan.FromMinutes(30), HDInsightClient.DefaultPollingInterval, GetAbstractionContext(), this.CreatingStates); var task = client.ListContainer(cluster.Name); task.WaitForResult(); var container = task.Result; Assert.IsNotNull(container); if (container.Error.IsNotNull()) { Assert.Fail("The Container was not expected to return an error but returned ({0}) ({1})", container.Error.HttpCode, container.Error.Message); } if (postCreateValidation != null) { postCreateValidation(container); } await client.DeleteContainer(cluster.Name); await client.WaitForClusterNullOrError(cluster.Name, cluster.Location, TimeSpan.FromMinutes(10), IntegrationTestBase.GetAbstractionContext().CancellationToken); Assert.IsNull(container.Error); Assert.IsNull(client.ListContainer(cluster.Name).WaitForResult()); } }
private static void ConfigHiveComponent(HiveComponent hive, ClusterCreateParametersV2 inputs) { hive.HiveSiteXmlProperties.AddRange( inputs.HiveConfiguration.ConfigurationCollection.Select(prop => new Property { Name = prop.Key, Value = prop.Value })); if (inputs.HiveConfiguration.AdditionalLibraries != null) { hive.AdditionalLibraries = new BlobContainerCredentialBackedResource() { AccountDnsName = inputs.HiveConfiguration.AdditionalLibraries.Name, BlobContainerName = inputs.HiveConfiguration.AdditionalLibraries.Container, Key = inputs.HiveConfiguration.AdditionalLibraries.Key }; } if (inputs.HiveMetastore != null) { hive.Metastore = new SqlAzureDatabaseCredentialBackedResource() { SqlServerName = inputs.HiveMetastore.Server, Credentials = new UsernamePasswordCredential() { Username = inputs.HiveMetastore.User, Password = inputs.HiveMetastore.Password }, DatabaseName = inputs.HiveMetastore.Database }; } }
public async Task ICanAddAndDeleteRDPUser() { IHDInsightSubscriptionCredentials credentials = IntegrationTestBase.GetValidCredentials(); IHDInsightClient client = HDInsightClient.Connect(credentials); ClusterCreateParametersV2 clusterCreationDetails = GetRandomCluster(); clusterCreationDetails.Version = "1.6"; ClusterDetails clusterDetails = client.CreateCluster(clusterCreationDetails); // add a rdp user - rdp is OFF by default, hence we enable it first string userName = "******"; string password = GetRandomValidPassword(); client.EnableRdp(clusterDetails.Name, clusterDetails.Location, userName, password, DateTime.UtcNow.AddHours(1)); ClusterDetails cluster = await client.GetClusterAsync(clusterDetails.Name); Assert.AreEqual(userName, cluster.RdpUserName, "Rdp user name has not been updated"); // now disable the rdp user await client.DisableRdpAsync(clusterDetails.Name, clusterDetails.Location); cluster = await client.GetClusterAsync(clusterDetails.Name); Assert.IsFalse(String.IsNullOrEmpty(cluster.Name), "Cluster user name is empty, maybe cluster was not created."); Assert.IsTrue(String.IsNullOrEmpty(cluster.RdpUserName), "Rdp user name has not been cleared"); // delete the cluster if (!string.Equals(clusterDetails.Name, IntegrationTestBase.TestCredentials.WellKnownCluster.DnsName)) { client.DeleteCluster(clusterDetails.Name); } }
public static ClusterDetails CreateClusterIfNotExists(ClusterCreateParametersV2 parameters) { try { LOG.InfoFormat("Checking if the HDInsight cluster exists - SubscriptionId: {0}, Name: {1}", AppConfig.SubscriptionId, parameters.Name); var cluster = Client.GetCluster(parameters.Name); if (cluster == null) { LOG.InfoFormat("Submitting a new HDInsight cluster create request - - SubscriptionId: {0}, Name: {1}", AppConfig.SubscriptionId, parameters.Name); cluster = Client.CreateCluster(parameters); } else { LOG.InfoFormat("Skipping creation, found an existing HDInsight Cluster with same name. Name: {0}", parameters.Name); } if (cluster.State != ClusterState.Operational && cluster.State != ClusterState.Running) { throw new ApplicationException( String.Format("HDInsight Cluster is not in a operational or running state. Name: {0}, State: {1}", cluster.Name, cluster.StateString)); } LOG.InfoFormat("HDInsight cluster created. Details:\r\n{0}", ClusterDetailsAsString(cluster)); return(cluster); } catch (Exception ex) { LOG.Error( String.Format("Failed to create cluster - SubscriptionId: {0}, Name: {1}", AppConfig.SubscriptionId, parameters.Name), ex); throw; } }
/// <summary> /// Converts user supplied cluster create parameters to IaasCluster type as understood by the Resource Provider. /// </summary> /// <param name="clusterCreateParameters">User supplied cluster create parameters.</param> /// <param name="userSubscriptionId">User's subscription Id.</param> /// <returns>An Instance of IaasCluster</returns> public static IaasCluster ConvertToIaasCluster(ClusterCreateParametersV2 clusterCreateParameters, string userSubscriptionId) { if (clusterCreateParameters == null) { throw new ArgumentNullException("clusterCreateParameters"); } if (String.IsNullOrEmpty(userSubscriptionId)) { throw new ArgumentNullException("userSubscriptionId"); } var correlationId = Guid.NewGuid().ToString(); var iaasCluster = new IaasCluster() { Id = clusterCreateParameters.Name, Location = clusterCreateParameters.Location, ApiVersion = "1.0", UserSubscriptionId = userSubscriptionId, UserTags = new Dictionary <string, string>(), HdiVersion = clusterCreateParameters.Version, DeploymentDocuments = new Dictionary <string, string>() { { IaasClusterDocumentTypes.EmbeddedAmbariConfigurationDocument, GenerateAmbariConfigurationDocument(clusterCreateParameters) }, { IaasClusterDocumentTypes.EmbeddedAzureConfigurationDocument, GenerateAzureDocument(clusterCreateParameters) } } }; iaasCluster.UserTags.Add("Client", string.Format("HDInsight .Net SDK {0}", Assembly.GetExecutingAssembly().GetName().Version.ToString())); return(iaasCluster); }
private static string GenerateAzureDocument(ClusterCreateParametersV2 clusterCreateParameters) { if (clusterCreateParameters == null) { throw new ArgumentNullException("clusterCreateParameters"); } string document = VersionToDocumentMapper.GetAzureCsmDocument(!String.IsNullOrEmpty(clusterCreateParameters.SshPassword)); AzureCsmDocumentManager azureCsmDocumentManager = new AzureCsmDocumentManager(document); // Set basic cluster parameters azureCsmDocumentManager.SetDnsName(clusterCreateParameters.Name); azureCsmDocumentManager.SetLocation(clusterCreateParameters.Location); azureCsmDocumentManager.SetWorkerNodeCount(clusterCreateParameters.ClusterSizeInNodes); azureCsmDocumentManager.SetHeadNodeVMSize(String.IsNullOrEmpty(clusterCreateParameters.HeadNodeSize) ? NodeVMSize.Large.ToString() : clusterCreateParameters.HeadNodeSize); azureCsmDocumentManager.SetDataNodeVMSize(String.IsNullOrEmpty(clusterCreateParameters.DataNodeSize) ? NodeVMSize.Large.ToString() : clusterCreateParameters.DataNodeSize); // Set SSH parameters if (!String.IsNullOrEmpty(clusterCreateParameters.SshUserName)) { if (!String.IsNullOrEmpty(clusterCreateParameters.SshPassword)) { azureCsmDocumentManager.SetSshProfile(clusterCreateParameters.SshUserName, clusterCreateParameters.SshPassword); } else { var x509cert = GetOpenSshCertificate(clusterCreateParameters.SshPublicKey, String.Format("CN={0}", clusterCreateParameters.Name)); azureCsmDocumentManager.SetSshProfile(clusterCreateParameters.SshUserName, x509cert); } } return(azureCsmDocumentManager.Document); }
public ClusterDetails CreateCluster(ClusterCreateParametersV2 cluster) { Task <ClusterDetails> createTask = this.CreateClusterAsync(cluster); createTask.Wait(); return(createTask.Result); }
public static ClusterDetails CreateClusterIfNotExists(ClusterType clusterType) { var name = AppConfig.AzureResourceName + clusterType.ToString().ToLowerInvariant(); LOG.Info(String.Format("Creating Cluster: Name = {0}, Type = {1}", name, clusterType)); var parameters = new ClusterCreateParametersV2() { Name = name, ClusterSizeInNodes = HDInsightClusterSize, Location = AppConfig.AzureResourceLocation, ClusterType = clusterType, HeadNodeSize = "Large", DataNodeSize = "Large", ZookeeperNodeSize = "Medium", UserName = AppConfig.AzureResourceUsername, Password = AppConfig.AzureResourcePassword, DefaultStorageAccountName = AzureStorageHelper.Name, DefaultStorageAccountKey = AzureStorageHelper.PrimaryKey, DefaultStorageContainer = AppConfig.AzureResourceName, Version = "3.2", //VirtualNetworkId = VirtualNetworkHelper.VNetId, //SubnetName = VirtualNetworkHelper.SubnetName, }; return(CreateClusterIfNotExists(parameters)); }
public override async Task EndProcessing() { IHDInsightClient client = this.GetClient(IgnoreSslErrors); client.ClusterProvisioning += this.ClientOnClusterProvisioning; ClusterCreateParametersV2 createClusterRequest = this.GetClusterCreateParameters(); var cluster = await client.CreateClusterAsync(createClusterRequest); this.Output.Add(new AzureHDInsightCluster(cluster)); }
private static WabStorageAccountConfiguration GetStorageAccountForScript(ScriptAction sa, ClusterCreateParametersV2 details) { var accts = new List<WabStorageAccountConfiguration>(); accts.Add(new WabStorageAccountConfiguration( details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer)); accts.AddRange(details.AdditionalStorageAccounts); // Tests whether the host for the script is in the list of provided storage accounts. var storage = (from acct in accts where GetFullyQualifiedStorageAccountName(acct.Name).Equals( sa.Uri.Host, StringComparison.OrdinalIgnoreCase) select acct).FirstOrDefault(); return storage; }
private async Task ValidateCreateClusterFailsWithError(ClusterCreateParametersV2 cluster) { IHDInsightCertificateCredential credentials = IntegrationTestBase.GetValidCredentials(); var client = ServiceLocator.Instance.Locate <IHDInsightManagementPocoClientFactory>().Create(credentials, GetAbstractionContext(), false); await client.CreateContainer(cluster); await client.WaitForClusterNotNull(cluster.Name, cluster.Location, TimeSpan.FromMinutes(10), GetAbstractionContext().CancellationToken); var result = await client.ListContainer(cluster.Name); Assert.IsNotNull(result); Assert.IsNotNull(result.Error); await client.DeleteContainer(cluster.Name); await client.WaitForClusterNull(cluster.Name, cluster.Location, TimeSpan.FromMinutes(10), GetAbstractionContext().CancellationToken); }
private static bool CreateHasNewVMSizesSpecified(ClusterCreateParametersV2 clusterCreateParameters) { return(new[] { clusterCreateParameters.HeadNodeSize, clusterCreateParameters.DataNodeSize, clusterCreateParameters.ZookeeperNodeSize } .Except( new[] { "ExtraLarge", "Large", "Medium", "Small", "ExtraSmall" }, StringComparer.OrdinalIgnoreCase).Any(ns => ns.IsNotNullOrEmpty())); }
private static string GenerateAmbariConfigurationDocument(ClusterCreateParametersV2 clusterCreateParameters) { if (clusterCreateParameters == null) { throw new ArgumentNullException("clusterCreateParameters"); } string document = VersionToDocumentMapper.GetAmbariConfigurationDocument(clusterCreateParameters.Version); AmbariConfigurationDocumentManager ambariConfigurationManager = new AmbariConfigurationDocumentManager(document); // Set password ambariConfigurationManager.SetPassword(clusterCreateParameters.Password); // Set default storage account information ambariConfigurationManager.RemoveStorageAccountEntries(); ambariConfigurationManager.SetDefaultStorageAccount(clusterCreateParameters.DefaultStorageContainer, clusterCreateParameters.DefaultStorageAccountName, clusterCreateParameters.DefaultStorageAccountKey); // Set additional storage accounts foreach (var storageAccount in clusterCreateParameters.AdditionalStorageAccounts) { ambariConfigurationManager.SetAdditionalStorageAccount(storageAccount.Name, storageAccount.Key); } // Set custom metastores if (clusterCreateParameters.HiveMetastore != null) { ambariConfigurationManager.SetCustomHiveMetastore(clusterCreateParameters.HiveMetastore); } if (clusterCreateParameters.OozieMetastore != null) { ambariConfigurationManager.SetCustomOozieMetastore(clusterCreateParameters.OozieMetastore); } // Set user specified Hadoop configurations ambariConfigurationManager.SetCustomConfigurations(AmbariConfigurationDocumentManager.CoreConfigurationKeyName, clusterCreateParameters.CoreConfiguration); ambariConfigurationManager.SetCustomConfigurations(AmbariConfigurationDocumentManager.HdfsConfigurationKeyName, clusterCreateParameters.HdfsConfiguration); ambariConfigurationManager.SetCustomConfigurations(AmbariConfigurationDocumentManager.YarnConfigurationKeyName, clusterCreateParameters.YarnConfiguration); ambariConfigurationManager.SetCustomConfigurations(AmbariConfigurationDocumentManager.HiveConfigurationKeyName, clusterCreateParameters.HiveConfiguration.ConfigurationCollection); ambariConfigurationManager.SetCustomConfigurations(AmbariConfigurationDocumentManager.OozieConfigurationKeyName, clusterCreateParameters.OozieConfiguration.ConfigurationCollection); ambariConfigurationManager.SetCustomConfigurations(AmbariConfigurationDocumentManager.MapredConfigurationKeyName, clusterCreateParameters.MapReduceConfiguration.ConfigurationCollection); return(ambariConfigurationManager.Document); }
public Task<ClusterDetails> CreateClusterAsync(ClusterCreateParametersV2 clusterCreateParameters) { this.LogMessage("Creating cluster '{0}' in location {1}", clusterCreateParameters.Name, clusterCreateParameters.Location); LastCreateRequest = clusterCreateParameters; var clusterDetails = new ClusterDetails(); clusterDetails.Name = clusterCreateParameters.Name; clusterDetails.HttpPassword = clusterCreateParameters.Password; clusterDetails.HttpUserName = clusterCreateParameters.UserName; clusterDetails.Version = clusterCreateParameters.Version; clusterDetails.Location = clusterCreateParameters.Location; clusterDetails.State = ClusterState.Running; clusterDetails.AdditionalStorageAccounts = clusterCreateParameters.AdditionalStorageAccounts; clusterDetails.DefaultStorageAccount = new WabStorageAccountConfiguration( clusterCreateParameters.DefaultStorageAccountName, clusterCreateParameters.DefaultStorageAccountKey, clusterCreateParameters.DefaultStorageContainer); Clusters.Add(new SimulatorClusterContainer { Cluster = clusterDetails }); return TaskEx2.FromResult(clusterDetails); }
/// <summary> /// Validates, appends the FQDN suffix if required to storage accounts and creates the default cluster specified in <paramref name="details"/>. /// </summary> /// <param name="details">The details.</param> public static void ValidateAndResolveAsvAccountsAndPrep(ClusterCreateParametersV2 details) { var defaultStorageAccount = new WabStorageAccountConfiguration( details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer); // Flattens all the configurations into a single list for more uniform validation var asvList = ResolveStorageAccounts(details.AdditionalStorageAccounts).ToList(); asvList.Add(ResolveStorageAccount(defaultStorageAccount)); // Basic validation on the ASV configurations if (string.IsNullOrEmpty(details.DefaultStorageContainer)) { throw new InvalidOperationException("Invalid Container. Default Storage Account Container cannot be null or empty"); } if (asvList.Any(asv => string.IsNullOrEmpty(asv.Name) || string.IsNullOrEmpty(asv.Key))) { throw new InvalidOperationException("Invalid Azure Configuration. Credentials cannot be null or empty"); } if (asvList.GroupBy(asv => asv.Name).Count(group => group.Count() > 1) > 0) { throw new InvalidOperationException("Invalid Azure Storage credential. Duplicated values detected"); } // Validates that we can establish the connection to the ASV Names and the default container var client = ServiceLocator.Instance.Locate <IAsvValidatorClientFactory>().Create(); asvList.ForEach(asv => client.ValidateAccount(asv.Name, asv.Key).Wait()); var resolvedAccounts = ResolveStorageAccounts(details.AdditionalStorageAccounts); details.AdditionalStorageAccounts.Clear(); foreach (var resolvedAccount in resolvedAccounts) { details.AdditionalStorageAccounts.Add(resolvedAccount); } var resolvedDefaultStorageAccount = ResolveStorageAccount(defaultStorageAccount); details.DefaultStorageAccountName = resolvedDefaultStorageAccount.Name; client.CreateContainerIfNotExists(details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer).Wait(); }
/// <summary> /// Validates, appends the FQDN suffix if required to storage accounts and creates the default cluster specified in <paramref name="details"/>. /// </summary> /// <param name="details">The details.</param> public static void ValidateAndResolveAsvAccountsAndPrep(ClusterCreateParametersV2 details) { var defaultStorageAccount = new WabStorageAccountConfiguration( details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer); // Flattens all the configurations into a single list for more uniform validation var asvList = ResolveStorageAccounts(details.AdditionalStorageAccounts).ToList(); asvList.Add(ResolveStorageAccount(defaultStorageAccount)); // Basic validation on the ASV configurations if (string.IsNullOrEmpty(details.DefaultStorageContainer)) { throw new InvalidOperationException("Invalid Container. Default Storage Account Container cannot be null or empty"); } if (asvList.Any(asv => string.IsNullOrEmpty(asv.Name) || string.IsNullOrEmpty(asv.Key))) { throw new InvalidOperationException("Invalid Azure Configuration. Credentials cannot be null or empty"); } if (asvList.GroupBy(asv => asv.Name).Count(group => group.Count() > 1) > 0) { throw new InvalidOperationException("Invalid Azure Storage credential. Duplicated values detected"); } // Validates that we can establish the connection to the ASV Names and the default container var client = ServiceLocator.Instance.Locate<IAsvValidatorClientFactory>().Create(); asvList.ForEach(asv => client.ValidateAccount(asv.Name, asv.Key).Wait()); var resolvedAccounts = ResolveStorageAccounts(details.AdditionalStorageAccounts); details.AdditionalStorageAccounts.Clear(); foreach (var resolvedAccount in resolvedAccounts) { details.AdditionalStorageAccounts.Add(resolvedAccount); } var resolvedDefaultStorageAccount = ResolveStorageAccount(defaultStorageAccount); details.DefaultStorageAccountName = resolvedDefaultStorageAccount.Name; client.CreateContainerIfNotExists(details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer).Wait(); }
public static void ValidateAndResolveConfigActionEndpointUris(ClusterCreateParametersV2 details) { if (details == null || details.ConfigActions == null) { return; } foreach (ConfigAction ca in details.ConfigActions) { ScriptAction sa = ca as ScriptAction; if (sa == null) { continue; } // Basic validation on the script action URI. if (sa.Uri == null || string.IsNullOrEmpty(sa.Uri.AbsoluteUri)) { throw new InvalidOperationException("Invalid Container. Script action URI cannot be null or empty"); } var storageAccount = GetStorageAccountForScript(sa, details); if (storageAccount != null) { // Check if the URI is in one of the provided storage accounts and whether it is reachable. ValidateAndResolveWasbScriptActionEndpointUri(sa.Uri, storageAccount).Wait(); } else { // Check if the URI is publicly reachable in Http format. ValidateAndResolveHttpScriptActionEndpointUri(sa.Uri).Wait(); } } }
private void fixDefaultExpectedZookeeperSize(ClusterCreateParametersV2 expected) { if (expected.ClusterType == ClusterType.HBase) { expected.ZookeeperNodeSize = "Medium"; } else { expected.ZookeeperNodeSize = "Small"; } }
internal ClusterCreateParametersV2 GetClusterCreateParameters() { var createClusterRequest = new ClusterCreateParametersV2(); createClusterRequest.Name = this.Name; createClusterRequest.Version = this.Version; createClusterRequest.Location = this.Location; createClusterRequest.CoreConfiguration.AddRange(this.CoreConfiguration); createClusterRequest.YarnConfiguration.AddRange(this.YarnConfiguration); createClusterRequest.HdfsConfiguration.AddRange(this.HdfsConfiguration); createClusterRequest.MapReduceConfiguration.ConfigurationCollection.AddRange(this.MapReduceConfiguration.ConfigurationCollection); createClusterRequest.MapReduceConfiguration.CapacitySchedulerConfigurationCollection.AddRange( this.MapReduceConfiguration.CapacitySchedulerConfigurationCollection); createClusterRequest.HiveConfiguration.AdditionalLibraries = this.HiveConfiguration.AdditionalLibraries; createClusterRequest.HiveConfiguration.ConfigurationCollection.AddRange(this.HiveConfiguration.ConfigurationCollection); createClusterRequest.OozieConfiguration.ConfigurationCollection.AddRange(this.OozieConfiguration.ConfigurationCollection); createClusterRequest.OozieConfiguration.AdditionalSharedLibraries = this.OozieConfiguration.AdditionalSharedLibraries; createClusterRequest.OozieConfiguration.AdditionalActionExecutorLibraries = this.OozieConfiguration.AdditionalActionExecutorLibraries; createClusterRequest.StormConfiguration.AddRange(this.StormConfiguration); createClusterRequest.SparkConfiguration.AddRange(this.SparkConfiguration); createClusterRequest.HBaseConfiguration.AdditionalLibraries = this.HBaseConfiguration.AdditionalLibraries; createClusterRequest.HBaseConfiguration.ConfigurationCollection.AddRange(this.HBaseConfiguration.ConfigurationCollection); createClusterRequest.DefaultStorageAccountName = this.DefaultStorageAccountName; createClusterRequest.DefaultStorageAccountKey = this.DefaultStorageAccountKey; createClusterRequest.DefaultStorageContainer = this.DefaultStorageContainerName; createClusterRequest.UserName = this.Credential.UserName; createClusterRequest.Password = this.Credential.GetCleartextPassword(); createClusterRequest.ClusterSizeInNodes = this.ClusterSizeInNodes; createClusterRequest.ClusterType = this.ClusterType; if (!string.IsNullOrEmpty(this.VirtualNetworkId)) { createClusterRequest.VirtualNetworkId = this.VirtualNetworkId; } if (!string.IsNullOrEmpty(this.SubnetName)) { createClusterRequest.SubnetName = this.SubnetName; } createClusterRequest.AdditionalStorageAccounts.AddRange( this.AdditionalStorageAccounts.Select(act => new WabStorageAccountConfiguration(act.StorageAccountName, act.StorageAccountKey))); createClusterRequest.ConfigActions.AddRange(this.ConfigActions.Select(ca => ca.ToSDKConfigAction())); if (this.HiveMetastore.IsNotNull()) { createClusterRequest.HiveMetastore = new Metastore( this.HiveMetastore.SqlAzureServerName, this.HiveMetastore.DatabaseName, this.HiveMetastore.Credential.UserName, this.HiveMetastore.Credential.GetCleartextPassword()); } if (this.OozieMetastore.IsNotNull()) { createClusterRequest.OozieMetastore = new Metastore( this.OozieMetastore.SqlAzureServerName, this.OozieMetastore.DatabaseName, this.OozieMetastore.Credential.UserName, this.OozieMetastore.Credential.GetCleartextPassword()); } if (!string.IsNullOrEmpty(this.HeadNodeSize) && !this.HeadNodeSize.Equals("Default", StringComparison.OrdinalIgnoreCase)) { createClusterRequest.HeadNodeSize = this.HeadNodeSize; } if (!string.IsNullOrEmpty(this.DataNodeSize)) { createClusterRequest.DataNodeSize = this.DataNodeSize; } if (!string.IsNullOrEmpty(this.ZookeeperNodeSize)) { createClusterRequest.ZookeeperNodeSize = this.ZookeeperNodeSize; } if (this.RdpCredential.IsNotNull()) { createClusterRequest.RdpUsername = this.RdpCredential.UserName; createClusterRequest.RdpPassword = this.RdpCredential.GetCleartextPassword(); } if (RdpAccessExpiry.IsNotNull()) { createClusterRequest.RdpAccessExpiry = this.RdpAccessExpiry; } // Set IaaS specific parameters createClusterRequest.OSType = this.OSType; if (SshCredential != null) { createClusterRequest.SshUserName = this.SshCredential.UserName; createClusterRequest.SshPassword = this.SshCredential.GetCleartextPassword(); } if (!string.IsNullOrEmpty(SshPublicKey)) { createClusterRequest.SshPublicKey = this.SshPublicKey; } return(createClusterRequest); }
/// <summary> /// Creates the wire contract request from user's cluster create parameters. /// </summary> /// <param name="cluster">The cluster.</param> /// <returns>An Instance of cluster create parameters.</returns> public static ClusterCreateParameters CreateWireClusterCreateParametersFromUserType(ClusterCreateParametersV2 cluster) { if (cluster == null) { throw new ArgumentNullException("cluster"); } ClusterCreateParameters ccp = null; if (cluster.Version.Equals("default", StringComparison.OrdinalIgnoreCase) || new Version(cluster.Version).Major >= 3) { switch (cluster.ClusterType) { case ClusterType.HBase: ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndHBaseTemplate(cluster); break; case ClusterType.Storm: ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndStormTemplate(cluster); break; case ClusterType.Hadoop: ccp = HDInsightClusterRequestGenerator.Create3XClusterFromMapReduceTemplate(cluster); break; case ClusterType.Spark: ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndSparkTemplate(cluster); break; default: throw new InvalidDataException( string.Format( CultureInfo.InvariantCulture, "Invalid cluster type '{0}' specified for cluster '{1}'", cluster.ClusterType, cluster.Name)); } } else { if (cluster.ClusterType != ClusterType.Hadoop) { throw new InvalidDataException(string.Format(CultureInfo.InvariantCulture, "Invalid cluster type '{0}' specified for cluster '{1}'", cluster.ClusterType, cluster.Name)); } ccp = new Version(cluster.Version).Major > 1 ? HDInsightClusterRequestGenerator.Create2XClusterForMapReduceTemplate(cluster) : HDInsightClusterRequestGenerator.Create1XClusterForMapReduceTemplate(cluster); } return(ccp); }
internal static void ValidateClusterConfiguration(ClusterDetails testCluster, ClusterCreateParametersV2 cluster) { var remoteCreds = new BasicAuthCredential() { Server = GatewayUriResolver.GetGatewayUri(new Uri(testCluster.ConnectionUrl).Host), UserName = testCluster.HttpUserName, Password = testCluster.HttpPassword }; var configurationAccessor = ServiceLocator.Instance.Locate <IAzureHDInsightClusterConfigurationAccessorFactory>().Create(remoteCreds); var coreConfiguration = configurationAccessor.GetCoreServiceConfiguration().WaitForResult(); ValidateConfiguration(cluster.CoreConfiguration, coreConfiguration); var hdfsConfiguration = configurationAccessor.GetHdfsServiceConfiguration().WaitForResult(); ValidateConfiguration(cluster.HdfsConfiguration, hdfsConfiguration); var mapReduceConfiguration = configurationAccessor.GetMapReduceServiceConfiguration().WaitForResult(); ValidateConfiguration(cluster.MapReduceConfiguration.ConfigurationCollection, mapReduceConfiguration); var hiveConfiguration = configurationAccessor.GetHiveServiceConfiguration().WaitForResult(); ValidateConfiguration(cluster.HiveConfiguration.ConfigurationCollection, hiveConfiguration); var oozieConfiguration = configurationAccessor.GetOozieServiceConfiguration().WaitForResult(); ValidateConfiguration(cluster.OozieConfiguration.ConfigurationCollection, oozieConfiguration); }
public void ICanCreateAClusterUsingPowerShellAndConfig_New_Set_Add_Hive_Oozie_And_CoreConfig() { AzureTestCredentials creds = GetCredentials(TestCredentialsNames.Default); string dnsName = this.GetRandomClusterName(); using (IRunspace runspace = this.GetPowerShellRunspace()) { IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.EndProcessing(); int expected = getCommand.Output.Count(); IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig) .WithParameter(CmdletConstants.ClusterSizeInNodes, 3) .AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage) .WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].DefaultStorageAccount.Name) .WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].DefaultStorageAccount.Key) .WithParameter(CmdletConstants.StorageContainerName, creds.Environments[0].DefaultStorageAccount.Container) .AddCommand(CmdletConstants.AddAzureHDInsightStorage) .WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].AdditionalStorageAccounts[0].Name) .WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].AdditionalStorageAccounts[0].Key) .AddCommand(CmdletConstants.AddAzureHDInsightMetastore) .WithParameter(CmdletConstants.SqlAzureServerName, creds.Environments[0].HiveStores[0].SqlServer) .WithParameter(CmdletConstants.DatabaseName, creds.Environments[0].HiveStores[0].Database) .WithParameter(CmdletConstants.Credential, GetPSCredential(creds.AzureUserName, creds.AzurePassword)) .WithParameter(CmdletConstants.MetastoreType, AzureHDInsightMetastoreType.HiveMetastore) .AddCommand(CmdletConstants.AddAzureHDInsightMetastore) .WithParameter(CmdletConstants.SqlAzureServerName, creds.Environments[0].OozieStores[0].SqlServer) .WithParameter(CmdletConstants.DatabaseName, creds.Environments[0].OozieStores[0].Database) .WithParameter(CmdletConstants.Credential, GetPSCredential(creds.AzureUserName, creds.AzurePassword)) .WithParameter(CmdletConstants.MetastoreType, AzureHDInsightMetastoreType.OozieMetastore) .AddCommand(CmdletConstants.NewAzureHDInsightCluster) .WithParameter(CmdletConstants.Name, dnsName) .WithParameter(CmdletConstants.Location, CmdletConstants.EastUs) .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword())) .Invoke(); ClusterCreateParametersV2 request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest; Assert.IsNotNull(request.HiveMetastore); Assert.IsNotNull(request.OozieMetastore); Assert.AreEqual(1, results.Results.Count); Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.Name = dnsName; getCommand.EndProcessing(); Assert.AreEqual(1, getCommand.Output.Count); Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name); results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster) .WithParameter(CmdletConstants.Name, dnsName) .Invoke(); Assert.AreEqual(0, results.Results.Count); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.EndProcessing(); Assert.AreEqual(expected, getCommand.Output.Count); } }
public async Task CreateContainer(ClusterCreateParametersV2 details) { this.LogMessage("Create Cluster Requested", Severity.Informational, Verbosity.Diagnostic); // Validates that the AzureStorage Configurations are valid and optionally append FQDN suffix to the storage account name AsvValidationHelper.ValidateAndResolveAsvAccountsAndPrep(details); // Validates that the config actions' Uris are downloadable. UriEndpointValidator.ValidateAndResolveConfigActionEndpointUris(details); var overrideHandlers = ServiceLocator.Instance.Locate <IHDInsightClusterOverrideManager>() .GetHandlers(this.credentials, this.Context, this.ignoreSslErrors); var rdfeCapabilitiesClient = ServiceLocator.Instance.Locate <IRdfeServiceRestClientFactory>() .Create(this.credentials, this.Context, this.ignoreSslErrors); var capabilities = await rdfeCapabilitiesClient.GetResourceProviderProperties(); if (!this.HasClusterCreateCapability(capabilities)) { throw new InvalidOperationException(string.Format( "Your subscription cannot create clusters, please contact Support")); } // For container resource type, config actions should never be enabled if (details.ConfigActions != null && details.ConfigActions.Count > 0) { throw new InvalidOperationException(string.Format( "Your subscription cannot create customized clusters, please contact Support")); } if (!new[] { "ExtraLarge", "Large" }.Contains(details.HeadNodeSize, StringComparer.OrdinalIgnoreCase) || !string.Equals(details.DataNodeSize, "Large", StringComparison.OrdinalIgnoreCase) || details.ZookeeperNodeSize.IsNotNullOrEmpty()) { throw new InvalidOperationException(string.Format( "Illegal node size specification for container resource. Headnode: [{0}], Datanode: [{1}], Zookeeper: [{2}]", details.HeadNodeSize, details.DataNodeSize, details.ZookeeperNodeSize)); } // Validates the region for the cluster creation var locationClient = ServiceLocator.Instance.Locate <ILocationFinderClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors); var availableLocations = locationClient.ListAvailableLocations(capabilities); if (!availableLocations.Contains(details.Location, StringComparer.OrdinalIgnoreCase)) { throw new InvalidOperationException(string.Format( "Cannot create a cluster in '{0}'. Available Locations for your subscription are: {1}", details.Location, string.Join(",", availableLocations))); } // Validates whether the subscription\location needs to be initialized var registrationClient = ServiceLocator.Instance.Locate <ISubscriptionRegistrationClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors); await registrationClient.RegisterSubscription(); if (!await registrationClient.ValidateSubscriptionLocation(details.Location)) { await registrationClient.RegisterSubscriptionLocation(details.Location); } // Creates the cluster var client = ServiceLocator.Instance.Locate <IHDInsightManagementRestClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors); if (details.ClusterType == ClusterType.HBase || details.ClusterType == ClusterType.Storm || details.ClusterType == ClusterType.Spark) { string payload = overrideHandlers.PayloadConverter.SerializeClusterCreateRequestV3(details); await client.CreateContainer(details.Name, details.Location, payload, 3); } else { if (!details.VirtualNetworkId.IsNullOrEmpty() || !details.SubnetName.IsNullOrEmpty()) { throw new InvalidOperationException("Create Hadoop clusters within a virtual network is not permitted."); } string payload = overrideHandlers.PayloadConverter.SerializeClusterCreateRequest(details); await client.CreateContainer(details.Name, details.Location, payload); } }
private async Task ValidateCreateClusterSucceeds(ClusterCreateParametersV2 cluster) { await ValidateCreateClusterSucceeds(cluster, null); }
private static WabStorageAccountConfiguration GetStorageAccountForScript(ScriptAction sa, ClusterCreateParametersV2 details) { var accts = new List <WabStorageAccountConfiguration>(); accts.Add(new WabStorageAccountConfiguration( details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer)); accts.AddRange(details.AdditionalStorageAccounts); // Tests whether the host for the script is in the list of provided storage accounts. var storage = (from acct in accts where GetFullyQualifiedStorageAccountName(acct.Name).Equals( sa.Uri.Host, StringComparison.OrdinalIgnoreCase) select acct).FirstOrDefault(); return(storage); }
public Task CreateContainer(ClusterCreateParametersV2 details) { this.LastCreateRequest = details; return(underlying.CreateContainer(details)); }
public ClusterDetails CreateCluster(ClusterCreateParametersV2 cluster, TimeSpan timeout) { return(this.CreateCluster(cluster)); }
public void ICanCreateAClusterUsingPowerShellAndConfig_New_Set_Add_ScriptAction() { AzureTestCredentials creds = GetCredentials(TestCredentialsNames.Default); string dnsName = this.GetRandomClusterName(); using (IRunspace runspace = this.GetPowerShellRunspace()) { IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.EndProcessing(); int expected = getCommand.Output.Count(); IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig) .WithParameter(CmdletConstants.ClusterSizeInNodes, 3) .AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage) .WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].DefaultStorageAccount.Name) .WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].DefaultStorageAccount.Key) .WithParameter(CmdletConstants.StorageContainerName, creds.Environments[0].DefaultStorageAccount.Container) .AddCommand(CmdletConstants.AddAzureHDInsightStorage) .WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].AdditionalStorageAccounts[0].Name) .WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].AdditionalStorageAccounts[0].Key) .AddCommand(CmdletConstants.AddAzureHDInsightScriptAction) .WithParameter(CmdletConstants.ConfigActionName, "test1") .WithParameter(CmdletConstants.ConfigActionClusterRoleCollection, "HeadNode") .WithParameter(CmdletConstants.ScriptActionUri, "http://test1.com") .WithParameter(CmdletConstants.ScriptActionParameters, "test1parameters") .AddCommand(CmdletConstants.AddAzureHDInsightScriptAction) .WithParameter(CmdletConstants.ConfigActionName, "test2") .WithParameter(CmdletConstants.ConfigActionClusterRoleCollection, "HeadNode") .WithParameter(CmdletConstants.ScriptActionUri, "http://test2.com") .WithParameter(CmdletConstants.ScriptActionParameters, "test2parameters") .AddCommand(CmdletConstants.NewAzureHDInsightCluster) .WithParameter(CmdletConstants.Name, dnsName) .WithParameter(CmdletConstants.Location, CmdletConstants.EastUs) .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword())) .Invoke(); ClusterCreateParametersV2 request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest; Assert.IsTrue(request.ConfigActions != null && request.ConfigActions.Count == 2); Assert.IsTrue( request.ConfigActions.ElementAt(0).Name == "test1" && request.ConfigActions.ElementAt(1).Name == "test2"); Assert.IsTrue( request.ConfigActions.ElementAt(0).ClusterRoleCollection.Count == 1 && request.ConfigActions.ElementAt(1).ClusterRoleCollection.Count == 1); Assert.AreEqual(1, results.Results.Count); Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.Name = dnsName; getCommand.EndProcessing(); Assert.AreEqual(1, getCommand.Output.Count); Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name); results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster) .WithParameter(CmdletConstants.Name, dnsName) .Invoke(); Assert.AreEqual(0, results.Results.Count); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.EndProcessing(); Assert.AreEqual(expected, getCommand.Output.Count); } }
/// <summary> /// Creates the wire contract request from user's cluster create parameters. /// </summary> /// <param name="cluster">The cluster.</param> /// <returns>An Instance of cluster create parameters.</returns> public static ClusterCreateParameters CreateWireClusterCreateParametersFromUserType(ClusterCreateParametersV2 cluster) { if (cluster == null) { throw new ArgumentNullException("cluster"); } ClusterCreateParameters ccp = null; if (cluster.Version.Equals("default", StringComparison.OrdinalIgnoreCase) || new Version(ClusterVersionUtils.TryGetVersionNumber(cluster.Version)).Major >= 3) { switch (cluster.ClusterType) { case ClusterType.HBase: ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndHBaseTemplate(cluster); break; case ClusterType.Storm: ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndStormTemplate(cluster); break; case ClusterType.Hadoop: ccp = HDInsightClusterRequestGenerator.Create3XClusterFromMapReduceTemplate(cluster); break; case ClusterType.Spark: ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndSparkTemplate(cluster); break; default: throw new InvalidDataException( string.Format( CultureInfo.InvariantCulture, "Invalid cluster type '{0}' specified for cluster '{1}'", cluster.ClusterType, cluster.Name)); } } else { if (cluster.ClusterType != ClusterType.Hadoop) { throw new InvalidDataException(string.Format(CultureInfo.InvariantCulture, "Invalid cluster type '{0}' specified for cluster '{1}'", cluster.ClusterType, cluster.Name)); } ccp = new Version(cluster.Version).Major > 1 ? HDInsightClusterRequestGenerator.Create2XClusterForMapReduceTemplate(cluster) : HDInsightClusterRequestGenerator.Create1XClusterForMapReduceTemplate(cluster); } return ccp; }