public void TestCreateHumboldtCluster()
{
string testName = "TestCreateHumboldtCluster";
ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-humboldt", parameters);
}
public void TestCreateWithDataLakeStorage()
{
string testName = "TestCreateWithDataLakeStorage";
ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersForAdl(testName);
RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-adl", parameters);
}
public void TestOMSOnRunningCluster()
{
string clusterName = "hdisdk-oms";
string testName = "TestOMSOnRunningCluster";
string suiteName = GetType().FullName;
ClusterCreateParameters createParams = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
createParams.Version = "3.6";
createParams.ClusterType = "Spark";
HDInsightManagementTestUtilities.CreateClusterInNewResourceGroupAndRunTest(suiteName, testName, clusterName, createParams, (client, rgName) =>
{
ClusterMonitoringRequest request = new ClusterMonitoringRequest
{
WorkspaceId = WorkspaceId,
PrimaryKey = PrimaryKey
};
client.Extensions.EnableMonitoring(rgName, clusterName, request);
ClusterMonitoringResponse monitoringStatus = client.Extensions.GetMonitoringStatus(rgName, clusterName);
Assert.True(monitoringStatus.ClusterMonitoringEnabled);
Assert.Equal(monitoringStatus.WorkspaceId, WorkspaceId);
client.Extensions.DisableMonitoring(rgName, clusterName);
monitoringStatus = client.Extensions.GetMonitoringStatus(rgName, clusterName);
Assert.False(monitoringStatus.ClusterMonitoringEnabled);
Assert.Null(monitoringStatus.WorkspaceId);
});
}
public void TestListClustersInResourceGroup()
{
string suiteName = GetType().FullName;
string testName = "TestListClustersInResourceGroup";
HDInsightManagementTestUtilities.RunTestInNewResourceGroup(suiteName, testName,
(resClient, client, rgName) =>
{
string clusterName1 = "hdisdk-cluster1";
string clusterName2 = "hdisdk-cluster2";
try
{
var list = client.Clusters.ListByResourceGroup(rgName);
Assert.DoesNotContain(list, c => c.Name.Equals(clusterName1, StringComparison.OrdinalIgnoreCase));
Assert.DoesNotContain(list, c => c.Name.Equals(clusterName2, StringComparison.OrdinalIgnoreCase));
// Create one cluster with ADLS so both clusters aren't using the same storage account at the same time
ClusterCreateParameters parameters1 = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
ClusterCreateParameters parameters2 = ClusterCreateParametersHelpers.GetCustomCreateParametersForAdl(testName);
Parallel.Invoke(
() => client.Clusters.Create(rgName, clusterName1, parameters1),
() => client.Clusters.Create(rgName, clusterName2, parameters2));
list = client.Clusters.ListByResourceGroup(rgName);
Assert.Contains(list, c => c.Name.Equals(clusterName1, StringComparison.OrdinalIgnoreCase));
Assert.Contains(list, c => c.Name.Equals(clusterName2, StringComparison.OrdinalIgnoreCase));
}
finally
{
client.Clusters.BeginDelete(rgName, clusterName1);
client.Clusters.BeginDelete(rgName, clusterName2);
}
});
}
public void TestHueOnRunningCluster()
{
string clusterName = "hdisdk-applications-hue";
string testName = "TestHueOnRunningCluster";
string suiteName = GetType().FullName;
ClusterCreateParameters createParams = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
createParams.Version = "3.6";
createParams.EdgeNodeSize = "Large";
HDInsightManagementTestUtilities.CreateClusterInNewResourceGroupAndRunTest(suiteName, testName, clusterName, createParams, (client, rgName) =>
{
string applicationName = "MyApplication";
Application application = new Application
{
Properties = new ApplicationProperties
{
InstallScriptActions = new[]
{
new RuntimeScriptAction
{
Name = "InstallHue",
Uri = "https://hdiconfigactions.blob.core.windows.net/linuxhueconfigactionv02/install-hue-uber-v02.sh",
Parameters = "-version latest -port 20000",
Roles = new [] { "edgenode" }
}
},
ApplicationType = "CustomApplication",
ComputeProfile = new ComputeProfile
{
Roles = new List <Role>
{
new Role
{
Name = "edgenode",
HardwareProfile = new HardwareProfile
{
VmSize = createParams.EdgeNodeSize
},
TargetInstanceCount = 1
}
}
}
}
};
client.Applications.Create(rgName, clusterName, applicationName, application);
IPage <Application> listApplications = client.Applications.ListByCluster(rgName, clusterName);
Assert.NotEmpty(listApplications);
Assert.Single(listApplications, app => app.Name.Equals(applicationName, StringComparison.OrdinalIgnoreCase));
client.Applications.Delete(rgName, clusterName, applicationName);
listApplications = client.Applications.ListByCluster(rgName, clusterName);
Assert.Empty(listApplications);
});
}
public void TestCreateRServerCluster()
{
string testName = "TestCreateRServerCluster";
ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
parameters.ClusterType = "RServer";
RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-rserver", parameters);
}
public void TestCreateMLServicesCluster()
{
string testName = "TestCreateMLServicesCluster";
ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
parameters.Version = "3.6";
parameters.ClusterType = "MLServices";
RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-mlservices", parameters);
}
public void TestCreateLinuxSparkClusterWithComponentVersion()
{
string testName = "TestCreateLinuxSparkClusterWithComponentVersion";
ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
parameters.ClusterType = "Spark";
parameters.ComponentVersion.Add("Spark", "2.2");
RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-sparkcomponentversions", parameters);
}
public void TestCreateHumboldtClusterWithCustomVMSizes()
{
string testName = "TestCreateHumboldtClusterWithCustomVMSizes";
ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
parameters.HeadNodeSize = "ExtraLarge";
parameters.ZookeeperNodeSize = "Medium";
RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-customvmsizes", parameters);
}
public void TestGetConfigurations()
{
string clusterName = "hdisdk-configs";
string testName = "TestGetConfigurations";
string suiteName = GetType().FullName;
ClusterCreateParameters createParams = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
Dictionary <string, string> hiveConfig = new Dictionary <string, string>
{
{ "key1", "value1" },
{ "key2", "value2" }
};
Dictionary <string, string> mapredConfig = new Dictionary <string, string>
{
{ "key5", "value5" },
{ "key6", "value6" }
};
Dictionary <string, string> yarnConfig = new Dictionary <string, string>
{
{ "key7", "value7" },
{ "key8", "value8" }
};
createParams.Configurations.Add(ConfigurationKey.HiveSite, hiveConfig);
createParams.Configurations.Add(ConfigurationKey.MapRedSite, mapredConfig);
createParams.Configurations.Add(ConfigurationKey.YarnSite, yarnConfig);
HDInsightManagementTestUtilities.CreateClusterInNewResourceGroupAndRunTest(suiteName, testName, clusterName, createParams, (client, rgName) =>
{
var hive = client.Configurations.Get(rgName, clusterName, ConfigurationKey.HiveSite);
Assert.Equal(hiveConfig, hive);
var mapred = client.Configurations.Get(rgName, clusterName, ConfigurationKey.MapRedSite);
Assert.Equal(mapredConfig, mapred);
var yarn = client.Configurations.Get(rgName, clusterName, ConfigurationKey.YarnSite);
Assert.Equal(yarnConfig, yarn);
var gateway = client.Configurations.Get(rgName, clusterName, ConfigurationKey.Gateway);
Assert.Equal(3, gateway.Count);
HttpConnectivitySettings settings = JsonConvert.DeserializeObject <HttpConnectivitySettings>(JsonConvert.SerializeObject(gateway));
Assert.NotNull(settings);
var core = client.Configurations.Get(rgName, clusterName, ConfigurationKey.CoreSite);
Assert.Equal(2, core.Count);
Assert.True(core.ContainsKey(Constants.StorageConfigurations.DefaultFsKey));
string storageKeyFormat = Constants.StorageConfigurations.WasbStorageAccountKeyFormat;
string storageKeyPrefix = storageKeyFormat.Substring(0, storageKeyFormat.IndexOf("{"));
Assert.Contains(core, c => c.Key.StartsWith(storageKeyPrefix));
});
}
public void TestCreateKafkaClusterWithManagedDisks()
{
string testName = "TestCreateKafkaClusterWithManagedDisks";
ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
parameters.ClusterType = "Kafka";
parameters.WorkerNodeDataDisksGroups = new List <DataDisksGroups>
{
new DataDisksGroups
{
DisksPerNode = 8
}
};
RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-kafka", parameters);
}
public void TestResizeCluster()
{
string clusterName = "hdisdk-clusterresize";
string testName = "TestResizeCluster";
string suiteName = GetType().FullName;
ClusterCreateParameters createParams = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
HDInsightManagementTestUtilities.CreateClusterInNewResourceGroupAndRunTest(suiteName, testName, clusterName, createParams, (client, rgName) =>
{
Cluster cluster = client.Clusters.Get(rgName, clusterName);
Assert.Equal(createParams.ClusterSizeInNodes, cluster.Properties.ComputeProfile.Roles.First(r => r.Name.Equals("workernode", StringComparison.OrdinalIgnoreCase)).TargetInstanceCount);
client.Clusters.Resize(rgName, clusterName, createParams.ClusterSizeInNodes + 1);
cluster = client.Clusters.Get(rgName, clusterName);
Assert.Equal(createParams.ClusterSizeInNodes + 1, cluster.Properties.ComputeProfile.Roles.First(r => r.Name.Equals("workernode", StringComparison.OrdinalIgnoreCase)).TargetInstanceCount);
});
}
public void TestCreateKafkaClusterWithDiskEncryption()
{
string testName = nameof(TestCreateKafkaClusterWithDiskEncryption);
ClusterCreateParameters parameters = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
parameters.ClusterType = "Kafka";
parameters.WorkerNodeDataDisksGroups = new List <DataDisksGroups>
{
new DataDisksGroups
{
DisksPerNode = 8
}
};
parameters.ClusterIdentity = ClusterCreateParametersHelpers.GetClusterIdentity();
parameters.DiskEncryptionProperties = ClusterCreateParametersHelpers.GetDiskEncryptionProperties();
RunCreateClusterTestInNewResourceGroup(GetType().FullName, testName, "hdisdk-kafka-byok", parameters);
}
public void TestHttpCustom()
{
string clusterName = "hdisdk-httpcustom";
string testName = "CanDisableEnableDisableHttpCustom";
string suiteName = GetType().FullName;
ClusterCreateParameters createParams = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
HDInsightManagementTestUtilities.CreateClusterInNewResourceGroupAndRunTest(suiteName, testName, clusterName, createParams, (client, rgName) =>
{
IDictionary <string, string> httpSettings = client.Configurations.Get(rgName, clusterName, ConfigurationKey.Gateway);
ValidateHttpSettings(httpSettings, createParams.UserName, createParams.Password);
string newPassword = "******";
client.Configurations.EnableHttp(rgName, clusterName, "admin", newPassword);
httpSettings = client.Configurations.Get(rgName, clusterName, ConfigurationKey.Gateway);
ValidateHttpSettings(httpSettings, createParams.UserName, newPassword);
});
}
public void TestListClustersInSubscription()
{
string suiteName = GetType().FullName;
string testName = "TestListClustersInSubscription";
using (MockContext context = MockContext.Start(suiteName, testName))
{
var handler = new RecordedDelegatingHandler {
StatusCodeToReturn = HttpStatusCode.OK
};
var resourceClient = HDInsightManagementTestUtilities.GetResourceManagementClient(context, handler);
var client = HDInsightManagementTestUtilities.GetHDInsightManagementClient(context, handler);
string rgName1 = string.Empty;
string rgName2 = string.Empty;
string clusterName1 = "hdisdk-cluster-rg1";
string clusterName2 = "hdisdk-cluster-rg2";
try
{
rgName1 = HDInsightManagementTestUtilities.CreateResourceGroup(resourceClient);
rgName2 = HDInsightManagementTestUtilities.CreateResourceGroup(resourceClient);
var list = client.Clusters.List();
Assert.DoesNotContain(list, c => c.Name.Equals(clusterName1, StringComparison.OrdinalIgnoreCase));
Assert.DoesNotContain(list, c => c.Name.Equals(clusterName2, StringComparison.OrdinalIgnoreCase));
// Create one cluster with ADLS so both clusters aren't using the same storage account at the same time
ClusterCreateParameters parameters1 = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
ClusterCreateParameters parameters2 = ClusterCreateParametersHelpers.GetCustomCreateParametersForAdl(testName);
Parallel.Invoke(
() => client.Clusters.Create(rgName1, clusterName1, parameters1),
() => client.Clusters.Create(rgName2, clusterName2, parameters2));
list = client.Clusters.List();
Assert.Contains(list, c => c.Name.Equals(clusterName1, StringComparison.OrdinalIgnoreCase));
Assert.Contains(list, c => c.Name.Equals(clusterName2, StringComparison.OrdinalIgnoreCase));
}
finally
{
resourceClient.ResourceGroups.BeginDelete(rgName1);
resourceClient.ResourceGroups.BeginDelete(rgName2);
}
}
}
public void TestHttpExtended()
{
string clusterName = "hdisdk-http";
string testName = "CanDisableEnableDisableHttpExtended";
string suiteName = GetType().FullName;
ClusterCreateParameters createParams = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
HDInsightManagementTestUtilities.CreateClusterInNewResourceGroupAndRunTest(suiteName, testName, clusterName, createParams, (client, rgName) =>
{
var httpSettings = client.Configurations.Get(rgName, clusterName, ConfigurationKey.Gateway);
ValidateHttpSettings(httpSettings, createParams.UserName, createParams.Password);
string newPassword = "******";
client.Configurations.Update(rgName, clusterName, ConfigurationKey.Gateway,
ConfigurationsConverter.Convert(new HttpConnectivitySettings
{
EnabledCredential = "true",
Username = "******",
Password = newPassword
}));
httpSettings = client.Configurations.Get(rgName, clusterName, ConfigurationKey.Gateway);
ValidateHttpSettings(httpSettings, createParams.UserName, newPassword);
});
}
public void TestScriptActionsOnRunningCluster()
{
string clusterName = "hdisdk-scriptactions";
string testName = "TestScriptActionsOnRunningCluster";
string suiteName = GetType().FullName;
ClusterCreateParameters createParams = ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName);
HDInsightManagementTestUtilities.CreateClusterInNewResourceGroupAndRunTest(suiteName, testName, clusterName, createParams, (client, rgName) =>
{
string scriptName = "script1";
//Upload script to storage account.
UploadScriptToStorageAccount(createParams);
//Execute script actions, and persist on success.
IList <RuntimeScriptAction> scriptActionParams = GetExecuteScriptActionParams(scriptName, InstallGiraph);
client.Clusters.ExecuteScriptActions(rgName, clusterName, scriptActionParams, true);
//List script actions and validate script is persisted.
IPage <RuntimeScriptActionDetail> scriptActionsList = client.ScriptActions.ListByCluster(rgName, clusterName);
Assert.Single(scriptActionsList);
RuntimeScriptActionDetail scriptAction = scriptActionsList.First();
Assert.Equal(scriptActionParams[0].Name, scriptAction.Name);
Assert.Equal(scriptActionParams[0].Uri, scriptAction.Uri);
Assert.Equal(scriptActionParams[0].Roles, scriptAction.Roles);
//Delete script action.
client.ScriptActions.Delete(rgName, clusterName, scriptName);
//List script actions and validate script is deleted.
scriptActionsList = client.ScriptActions.ListByCluster(rgName, clusterName);
Assert.Empty(scriptActionsList);
//List script action history and validate script appears there.
IPage <RuntimeScriptActionDetail> listHistoryResponse = client.ScriptExecutionHistory.ListByCluster(rgName, clusterName);
Assert.Single(listHistoryResponse);
scriptAction = listHistoryResponse.First();
Assert.Equal(1, scriptAction.ExecutionSummary.Count);
Assert.Equal(scriptActionParams[0].Name, scriptAction.Name);
Assert.Equal(scriptActionParams[0].Uri, scriptAction.Uri);
Assert.Equal(scriptActionParams[0].Roles, scriptAction.Roles);
Assert.Equal("Succeeded", scriptAction.Status);
//Get the script action by ID and validate it's the same action.
scriptAction = client.ScriptActions.GetExecutionDetail(rgName, clusterName, listHistoryResponse.First().ScriptExecutionId.Value.ToString());
Assert.Equal(scriptActionParams[0].Name, scriptAction.Name);
//Execute script actions, but don't persist on success.
scriptActionParams = GetExecuteScriptActionParams("script5baf", InstallGiraph);
client.Clusters.ExecuteScriptActions(rgName, clusterName, scriptActionParams, false);
//List script action history and validate the new script also appears.
listHistoryResponse = client.ScriptExecutionHistory.ListByCluster(rgName, clusterName);
Assert.Equal(2, listHistoryResponse.Count());
scriptAction = listHistoryResponse.FirstOrDefault(a => a.Name.Equals(scriptActionParams[0].Name, StringComparison.OrdinalIgnoreCase));
Assert.NotNull(scriptAction);
Assert.Equal(1, scriptAction.ExecutionSummary.Count);
Assert.Equal(scriptActionParams[0].Name, scriptAction.Name);
Assert.Equal(scriptActionParams[0].Uri, scriptAction.Uri);
Assert.Equal(scriptActionParams[0].Roles, scriptAction.Roles);
Assert.Equal("Succeeded", scriptAction.Status);
//Promote non-persisted script.
client.ScriptExecutionHistory.Promote(rgName, clusterName, listHistoryResponse.First().ScriptExecutionId.Value.ToString());
//Execute failing script action.
string failingScriptUri = GetFailingScriptUri(createParams);
IList <RuntimeScriptAction> failingScriptActionParams = GetExecuteScriptActionParams(string.Format("script0cc4", Guid.NewGuid()).Substring(0, 10), failingScriptUri);
CloudException ex = Assert.Throws <CloudException>(() => client.Clusters.ExecuteScriptActions(rgName, clusterName, failingScriptActionParams, true));
//List script action list and validate the promoted script is the only one there.
scriptActionsList = client.ScriptActions.ListByCluster(rgName, clusterName);
Assert.Single(scriptActionsList);
Assert.Equal(1, scriptAction.ExecutionSummary.Count);
Assert.Equal(scriptActionParams[0].Name, scriptAction.Name);
Assert.Equal(scriptActionParams[0].Uri, scriptAction.Uri);
Assert.Equal(scriptActionParams[0].Roles, scriptAction.Roles);
Assert.Equal("Succeeded", scriptAction.Status);
//List script action history and validate all three scripts are there.
listHistoryResponse = client.ScriptExecutionHistory.ListByCluster(rgName, clusterName);
Assert.Equal(3, listHistoryResponse.Count());
Assert.Equal(2, listHistoryResponse.Count(a => a.Status == "Succeeded"));
Assert.Equal(1, listHistoryResponse.Count(a => a.Status == "Failed"));
});
}
