public void TestCreateHumboldtCluster() { string testName = "TestCreateHumboldtCluster"; ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-humboldt", parameters); }
public void TestCreateWithDataLakeStorage() { string testName = "TestCreateWithDataLakeStorage"; ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersForAdl(testName); RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-adl", parameters); }
public void TestOMSOnRunningCluster() { string clusterName = "hdisdk-oms"; string testName = "TestOMSOnRunningCluster"; string suiteName = GetType().FullName; ClusterCreateParameters createParams = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); createParams.Version = "3.6"; createParams.ClusterType = "Spark"; HDInsightManagementTestUtilities.CreateClusterInNewResourceGroupAndRunTest(suiteName, testName, clusterName, createParams, (client, rgName) => { ClusterMonitoringRequest request = new ClusterMonitoringRequest { WorkspaceId = WorkspaceId, PrimaryKey = PrimaryKey }; client.Extensions.EnableMonitoring(rgName, clusterName, request); ClusterMonitoringResponse monitoringStatus = client.Extensions.GetMonitoringStatus(rgName, clusterName); Assert.True(monitoringStatus.ClusterMonitoringEnabled); Assert.Equal(monitoringStatus.WorkspaceId, WorkspaceId); client.Extensions.DisableMonitoring(rgName, clusterName); monitoringStatus = client.Extensions.GetMonitoringStatus(rgName, clusterName); Assert.False(monitoringStatus.ClusterMonitoringEnabled); Assert.Null(monitoringStatus.WorkspaceId); }); }
public void TestListClustersInResourceGroup() { string suiteName = GetType().FullName; string testName = "TestListClustersInResourceGroup"; HDInsightManagementTestUtilities.RunTestInNewResourceGroup(suiteName, testName, (resClient, client, rgName) => { string clusterName1 = "hdisdk-cluster1"; string clusterName2 = "hdisdk-cluster2"; try { var list = client.Clusters.ListByResourceGroup(rgName); Assert.DoesNotContain(list, c => c.Name.Equals(clusterName1, StringComparison.OrdinalIgnoreCase)); Assert.DoesNotContain(list, c => c.Name.Equals(clusterName2, StringComparison.OrdinalIgnoreCase)); // Create one cluster with ADLS so both clusters aren't using the same storage account at the same time ClusterCreateParameters parameters1 = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); ClusterCreateParameters parameters2 = ClusterCreateParametersHelpers.GetCustomCreateParametersForAdl(testName); Parallel.Invoke( () => client.Clusters.Create(rgName, clusterName1, parameters1), () => client.Clusters.Create(rgName, clusterName2, parameters2)); list = client.Clusters.ListByResourceGroup(rgName); Assert.Contains(list, c => c.Name.Equals(clusterName1, StringComparison.OrdinalIgnoreCase)); Assert.Contains(list, c => c.Name.Equals(clusterName2, StringComparison.OrdinalIgnoreCase)); } finally { client.Clusters.BeginDelete(rgName, clusterName1); client.Clusters.BeginDelete(rgName, clusterName2); } }); }
public void TestHueOnRunningCluster() { string clusterName = "hdisdk-applications-hue"; string testName = "TestHueOnRunningCluster"; string suiteName = GetType().FullName; ClusterCreateParameters createParams = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); createParams.Version = "3.6"; createParams.EdgeNodeSize = "Large"; HDInsightManagementTestUtilities.CreateClusterInNewResourceGroupAndRunTest(suiteName, testName, clusterName, createParams, (client, rgName) => { string applicationName = "MyApplication"; Application application = new Application { Properties = new ApplicationProperties { InstallScriptActions = new[] { new RuntimeScriptAction { Name = "InstallHue", Uri = "https://hdiconfigactions.blob.core.windows.net/linuxhueconfigactionv02/install-hue-uber-v02.sh", Parameters = "-version latest -port 20000", Roles = new [] { "edgenode" } } }, ApplicationType = "CustomApplication", ComputeProfile = new ComputeProfile { Roles = new List <Role> { new Role { Name = "edgenode", HardwareProfile = new HardwareProfile { VmSize = createParams.EdgeNodeSize }, TargetInstanceCount = 1 } } } } }; client.Applications.Create(rgName, clusterName, applicationName, application); IPage <Application> listApplications = client.Applications.ListByCluster(rgName, clusterName); Assert.NotEmpty(listApplications); Assert.Single(listApplications, app => app.Name.Equals(applicationName, StringComparison.OrdinalIgnoreCase)); client.Applications.Delete(rgName, clusterName, applicationName); listApplications = client.Applications.ListByCluster(rgName, clusterName); Assert.Empty(listApplications); }); }
public void TestCreateRServerCluster() { string testName = "TestCreateRServerCluster"; ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); parameters.ClusterType = "RServer"; RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-rserver", parameters); }
public void TestCreateMLServicesCluster() { string testName = "TestCreateMLServicesCluster"; ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); parameters.Version = "3.6"; parameters.ClusterType = "MLServices"; RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-mlservices", parameters); }
public void TestCreateLinuxSparkClusterWithComponentVersion() { string testName = "TestCreateLinuxSparkClusterWithComponentVersion"; ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); parameters.ClusterType = "Spark"; parameters.ComponentVersion.Add("Spark", "2.2"); RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-sparkcomponentversions", parameters); }
public void TestCreateHumboldtClusterWithCustomVMSizes() { string testName = "TestCreateHumboldtClusterWithCustomVMSizes"; ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); parameters.HeadNodeSize = "ExtraLarge"; parameters.ZookeeperNodeSize = "Medium"; RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-customvmsizes", parameters); }
public void TestGetConfigurations() { string clusterName = "hdisdk-configs"; string testName = "TestGetConfigurations"; string suiteName = GetType().FullName; ClusterCreateParameters createParams = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); Dictionary <string, string> hiveConfig = new Dictionary <string, string> { { "key1", "value1" }, { "key2", "value2" } }; Dictionary <string, string> mapredConfig = new Dictionary <string, string> { { "key5", "value5" }, { "key6", "value6" } }; Dictionary <string, string> yarnConfig = new Dictionary <string, string> { { "key7", "value7" }, { "key8", "value8" } }; createParams.Configurations.Add(ConfigurationKey.HiveSite, hiveConfig); createParams.Configurations.Add(ConfigurationKey.MapRedSite, mapredConfig); createParams.Configurations.Add(ConfigurationKey.YarnSite, yarnConfig); HDInsightManagementTestUtilities.CreateClusterInNewResourceGroupAndRunTest(suiteName, testName, clusterName, createParams, (client, rgName) => { var hive = client.Configurations.Get(rgName, clusterName, ConfigurationKey.HiveSite); Assert.Equal(hiveConfig, hive); var mapred = client.Configurations.Get(rgName, clusterName, ConfigurationKey.MapRedSite); Assert.Equal(mapredConfig, mapred); var yarn = client.Configurations.Get(rgName, clusterName, ConfigurationKey.YarnSite); Assert.Equal(yarnConfig, yarn); var gateway = client.Configurations.Get(rgName, clusterName, ConfigurationKey.Gateway); Assert.Equal(3, gateway.Count); HttpConnectivitySettings settings = JsonConvert.DeserializeObject <HttpConnectivitySettings>(JsonConvert.SerializeObject(gateway)); Assert.NotNull(settings); var core = client.Configurations.Get(rgName, clusterName, ConfigurationKey.CoreSite); Assert.Equal(2, core.Count); Assert.True(core.ContainsKey(Constants.StorageConfigurations.DefaultFsKey)); string storageKeyFormat = Constants.StorageConfigurations.WasbStorageAccountKeyFormat; string storageKeyPrefix = storageKeyFormat.Substring(0, storageKeyFormat.IndexOf("{")); Assert.Contains(core, c => c.Key.StartsWith(storageKeyPrefix)); }); }
public void TestCreateKafkaClusterWithManagedDisks() { string testName = "TestCreateKafkaClusterWithManagedDisks"; ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); parameters.ClusterType = "Kafka"; parameters.WorkerNodeDataDisksGroups = new List <DataDisksGroups> { new DataDisksGroups { DisksPerNode = 8 } }; RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-kafka", parameters); }
public void TestResizeCluster() { string clusterName = "hdisdk-clusterresize"; string testName = "TestResizeCluster"; string suiteName = GetType().FullName; ClusterCreateParameters createParams = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); HDInsightManagementTestUtilities.CreateClusterInNewResourceGroupAndRunTest(suiteName, testName, clusterName, createParams, (client, rgName) => { Cluster cluster = client.Clusters.Get(rgName, clusterName); Assert.Equal(createParams.ClusterSizeInNodes, cluster.Properties.ComputeProfile.Roles.First(r => r.Name.Equals("workernode", StringComparison.OrdinalIgnoreCase)).TargetInstanceCount); client.Clusters.Resize(rgName, clusterName, createParams.ClusterSizeInNodes + 1); cluster = client.Clusters.Get(rgName, clusterName); Assert.Equal(createParams.ClusterSizeInNodes + 1, cluster.Properties.ComputeProfile.Roles.First(r => r.Name.Equals("workernode", StringComparison.OrdinalIgnoreCase)).TargetInstanceCount); }); }
public void TestCreateKafkaClusterWithDiskEncryption() { string testName = nameof(TestCreateKafkaClusterWithDiskEncryption); ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); parameters.ClusterType = "Kafka"; parameters.WorkerNodeDataDisksGroups = new List <DataDisksGroups> { new DataDisksGroups { DisksPerNode = 8 } }; parameters.ClusterIdentity = ClusterCreateParametersHelpers.GetClusterIdentity(); parameters.DiskEncryptionProperties = ClusterCreateParametersHelpers.GetDiskEncryptionProperties(); RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-kafka-byok", parameters); }
public void TestHttpCustom() { string clusterName = "hdisdk-httpcustom"; string testName = "CanDisableEnableDisableHttpCustom"; string suiteName = GetType().FullName; ClusterCreateParameters createParams = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); HDInsightManagementTestUtilities.CreateClusterInNewResourceGroupAndRunTest(suiteName, testName, clusterName, createParams, (client, rgName) => { IDictionary <string, string> httpSettings = client.Configurations.Get(rgName, clusterName, ConfigurationKey.Gateway); ValidateHttpSettings(httpSettings, createParams.UserName, createParams.Password); string newPassword = "******"; client.Configurations.EnableHttp(rgName, clusterName, "admin", newPassword); httpSettings = client.Configurations.Get(rgName, clusterName, ConfigurationKey.Gateway); ValidateHttpSettings(httpSettings, createParams.UserName, newPassword); }); }
public void TestListClustersInSubscription() { string suiteName = GetType().FullName; string testName = "TestListClustersInSubscription"; using (MockContext context = MockContext.Start(suiteName, testName)) { var handler = new RecordedDelegatingHandler { StatusCodeToReturn = HttpStatusCode.OK }; var resourceClient = HDInsightManagementTestUtilities.GetResourceManagementClient(context, handler); var client = HDInsightManagementTestUtilities.GetHDInsightManagementClient(context, handler); string rgName1 = string.Empty; string rgName2 = string.Empty; string clusterName1 = "hdisdk-cluster-rg1"; string clusterName2 = "hdisdk-cluster-rg2"; try { rgName1 = HDInsightManagementTestUtilities.CreateResourceGroup(resourceClient); rgName2 = HDInsightManagementTestUtilities.CreateResourceGroup(resourceClient); var list = client.Clusters.List(); Assert.DoesNotContain(list, c => c.Name.Equals(clusterName1, StringComparison.OrdinalIgnoreCase)); Assert.DoesNotContain(list, c => c.Name.Equals(clusterName2, StringComparison.OrdinalIgnoreCase)); // Create one cluster with ADLS so both clusters aren't using the same storage account at the same time ClusterCreateParameters parameters1 = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); ClusterCreateParameters parameters2 = ClusterCreateParametersHelpers.GetCustomCreateParametersForAdl(testName); Parallel.Invoke( () => client.Clusters.Create(rgName1, clusterName1, parameters1), () => client.Clusters.Create(rgName2, clusterName2, parameters2)); list = client.Clusters.List(); Assert.Contains(list, c => c.Name.Equals(clusterName1, StringComparison.OrdinalIgnoreCase)); Assert.Contains(list, c => c.Name.Equals(clusterName2, StringComparison.OrdinalIgnoreCase)); } finally { resourceClient.ResourceGroups.BeginDelete(rgName1); resourceClient.ResourceGroups.BeginDelete(rgName2); } } }
public void TestHttpExtended() { string clusterName = "hdisdk-http"; string testName = "CanDisableEnableDisableHttpExtended"; string suiteName = GetType().FullName; ClusterCreateParameters createParams = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); HDInsightManagementTestUtilities.CreateClusterInNewResourceGroupAndRunTest(suiteName, testName, clusterName, createParams, (client, rgName) => { var httpSettings = client.Configurations.Get(rgName, clusterName, ConfigurationKey.Gateway); ValidateHttpSettings(httpSettings, createParams.UserName, createParams.Password); string newPassword = "******"; client.Configurations.Update(rgName, clusterName, ConfigurationKey.Gateway, ConfigurationsConverter.Convert(new HttpConnectivitySettings { EnabledCredential = "true", Username = "******", Password = newPassword })); httpSettings = client.Configurations.Get(rgName, clusterName, ConfigurationKey.Gateway); ValidateHttpSettings(httpSettings, createParams.UserName, newPassword); }); }
public void TestScriptActionsOnRunningCluster() { string clusterName = "hdisdk-scriptactions"; string testName = "TestScriptActionsOnRunningCluster"; string suiteName = GetType().FullName; ClusterCreateParameters createParams = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName); HDInsightManagementTestUtilities.CreateClusterInNewResourceGroupAndRunTest(suiteName, testName, clusterName, createParams, (client, rgName) => { string scriptName = "script1"; //Upload script to storage account. UploadScriptToStorageAccount(createParams); //Execute script actions, and persist on success. IList <RuntimeScriptAction> scriptActionParams = GetExecuteScriptActionParams(scriptName, InstallGiraph); client.Clusters.ExecuteScriptActions(rgName, clusterName, scriptActionParams, true); //List script actions and validate script is persisted. IPage <RuntimeScriptActionDetail> scriptActionsList = client.ScriptActions.ListByCluster(rgName, clusterName); Assert.Single(scriptActionsList); RuntimeScriptActionDetail scriptAction = scriptActionsList.First(); Assert.Equal(scriptActionParams[0].Name, scriptAction.Name); Assert.Equal(scriptActionParams[0].Uri, scriptAction.Uri); Assert.Equal(scriptActionParams[0].Roles, scriptAction.Roles); //Delete script action. client.ScriptActions.Delete(rgName, clusterName, scriptName); //List script actions and validate script is deleted. scriptActionsList = client.ScriptActions.ListByCluster(rgName, clusterName); Assert.Empty(scriptActionsList); //List script action history and validate script appears there. IPage <RuntimeScriptActionDetail> listHistoryResponse = client.ScriptExecutionHistory.ListByCluster(rgName, clusterName); Assert.Single(listHistoryResponse); scriptAction = listHistoryResponse.First(); Assert.Equal(1, scriptAction.ExecutionSummary.Count); Assert.Equal(scriptActionParams[0].Name, scriptAction.Name); Assert.Equal(scriptActionParams[0].Uri, scriptAction.Uri); Assert.Equal(scriptActionParams[0].Roles, scriptAction.Roles); Assert.Equal("Succeeded", scriptAction.Status); //Get the script action by ID and validate it's the same action. scriptAction = client.ScriptActions.GetExecutionDetail(rgName, clusterName, listHistoryResponse.First().ScriptExecutionId.Value.ToString()); Assert.Equal(scriptActionParams[0].Name, scriptAction.Name); //Execute script actions, but don't persist on success. scriptActionParams = GetExecuteScriptActionParams("script5baf", InstallGiraph); client.Clusters.ExecuteScriptActions(rgName, clusterName, scriptActionParams, false); //List script action history and validate the new script also appears. listHistoryResponse = client.ScriptExecutionHistory.ListByCluster(rgName, clusterName); Assert.Equal(2, listHistoryResponse.Count()); scriptAction = listHistoryResponse.FirstOrDefault(a => a.Name.Equals(scriptActionParams[0].Name, StringComparison.OrdinalIgnoreCase)); Assert.NotNull(scriptAction); Assert.Equal(1, scriptAction.ExecutionSummary.Count); Assert.Equal(scriptActionParams[0].Name, scriptAction.Name); Assert.Equal(scriptActionParams[0].Uri, scriptAction.Uri); Assert.Equal(scriptActionParams[0].Roles, scriptAction.Roles); Assert.Equal("Succeeded", scriptAction.Status); //Promote non-persisted script. client.ScriptExecutionHistory.Promote(rgName, clusterName, listHistoryResponse.First().ScriptExecutionId.Value.ToString()); //Execute failing script action. string failingScriptUri = GetFailingScriptUri(createParams); IList <RuntimeScriptAction> failingScriptActionParams = GetExecuteScriptActionParams(string.Format("script0cc4", Guid.NewGuid()).Substring(0, 10), failingScriptUri); CloudException ex = Assert.Throws <CloudException>(() => client.Clusters.ExecuteScriptActions(rgName, clusterName, failingScriptActionParams, true)); //List script action list and validate the promoted script is the only one there. scriptActionsList = client.ScriptActions.ListByCluster(rgName, clusterName); Assert.Single(scriptActionsList); Assert.Equal(1, scriptAction.ExecutionSummary.Count); Assert.Equal(scriptActionParams[0].Name, scriptAction.Name); Assert.Equal(scriptActionParams[0].Uri, scriptAction.Uri); Assert.Equal(scriptActionParams[0].Roles, scriptAction.Roles); Assert.Equal("Succeeded", scriptAction.Status); //List script action history and validate all three scripts are there. listHistoryResponse = client.ScriptExecutionHistory.ListByCluster(rgName, clusterName); Assert.Equal(3, listHistoryResponse.Count()); Assert.Equal(2, listHistoryResponse.Count(a => a.Status == "Succeeded")); Assert.Equal(1, listHistoryResponse.Count(a => a.Status == "Failed")); }); }