private void SerializeOozieConfiguration(ClusterCreateParametersV2 cluster, dynamic dynaXml)
{
this.AddConfigurationOptions(dynaXml, cluster.OozieConfiguration.ConfigurationCollection, "ooziesettings");
if (cluster.OozieConfiguration.AdditionalSharedLibraries != null)
{
dynaXml.rp("oozieadditionalsharedlibraries")
.AdditionalSharedLibraries
.b
.AccountName(cluster.OozieConfiguration.AdditionalSharedLibraries.Name)
.BlobContainerName(cluster.OozieConfiguration.AdditionalSharedLibraries.Container)
.Key(cluster.OozieConfiguration.AdditionalSharedLibraries.Key)
.d.End();
}
if (cluster.OozieConfiguration.AdditionalActionExecutorLibraries != null)
{
dynaXml.rp("ooziesharedexecutables")
.AdditionalActionExecutorLibraries
.b
.AccountName(cluster.OozieConfiguration.AdditionalActionExecutorLibraries.Name)
.BlobContainerName(cluster.OozieConfiguration.AdditionalActionExecutorLibraries.Container)
.Key(cluster.OozieConfiguration.AdditionalActionExecutorLibraries.Key)
.d.End();
}
this.SerializeOozieMetastore(cluster, dynaXml);
}
private static void HandleCreateHttpLayerException(ClusterCreateParametersV2 clusterCreateParameters, HttpLayerException e)
{
if (e.RequestContent.Contains(ClusterAlreadyExistsError) && e.RequestStatusCode == HttpStatusCode.BadRequest)
{
throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "Cluster {0} already exists.", clusterCreateParameters.Name));
}
}
/// <inheritdoc />
public string SerializeClusterCreateRequestV3(ClusterCreateParametersV2 cluster)
{
Contracts.May2014.ClusterCreateParameters ccp = null;
if (cluster.ClusterType == ClusterType.HBase)
{
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndHBaseTemplate(cluster);
}
else if (cluster.ClusterType == ClusterType.Storm)
{
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndStormTemplate(cluster);
}
else if (cluster.ClusterType == ClusterType.Spark)
{
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndSparkTemplate(cluster);
}
else if (cluster.ClusterType == ClusterType.Hadoop)
{
ccp = HDInsightClusterRequestGenerator.Create3XClusterFromMapReduceTemplate(cluster);
}
else
{
throw new InvalidDataException("Invalid cluster type");
}
return(this.CreateClusterRequest_ToInternalV3(ccp));
}
/// <inheritdoc />
public async Task <ClusterDetails> CreateClusterAsync(ClusterCreateParametersV2 clusterCreateParameters)
{
if (clusterCreateParameters.OSType == OSType.Linux)
{
return(await this.CreateIaasClusterAsync(clusterCreateParameters));
}
else
{
return(await this.CreatePaasClusterAsync(clusterCreateParameters));
}
}
public async Task <ClusterDetails> CreateClusterAsync(ClusterCreateParameters clusterCreateParameters)
{
if (clusterCreateParameters == null)
{
throw new ArgumentNullException("clusterCreateParameters");
}
var createParamsV2 = new ClusterCreateParametersV2(clusterCreateParameters);
return(await CreateClusterAsync(createParamsV2));
}
private void SerializeOozieMetastore(ClusterCreateParametersV2 cluster, dynamic dynaXml)
{
if (cluster.OozieMetastore != null)
{
dynaXml.rp("ooziesettings")
.Catalog.b.DatabaseName(cluster.OozieMetastore.Database)
.Password(cluster.OozieMetastore.Password)
.Server(cluster.OozieMetastore.Server)
.Username(cluster.OozieMetastore.User)
.d.End();
}
}
private bool createHasNewVMSizesSpecified(ClusterCreateParametersV2 clusterCreateParameters)
{
const string ExtraLarge = "ExtraLarge";
const string Large = "Large";
if (!new[] { Large, ExtraLarge }.Contains(clusterCreateParameters.HeadNodeSize, StringComparer.OrdinalIgnoreCase))
{
return(true);
}
if (!clusterCreateParameters.DataNodeSize.Equals(Large))
{
return(true);
}
return(clusterCreateParameters.ZookeeperNodeSize != null);
}
private void SerializeHiveConfiguration(ClusterCreateParametersV2 cluster, dynamic dynaXml)
{
if (cluster.HiveConfiguration.AdditionalLibraries != null)
{
dynaXml.rp("hiveresources")
.AdditionalLibraries.b.AccountName(cluster.HiveConfiguration.AdditionalLibraries.Name)
.BlobContainerName(cluster.HiveConfiguration.AdditionalLibraries.Container)
.Key(cluster.HiveConfiguration.AdditionalLibraries.Key)
.d.End();
}
if (cluster.HiveMetastore != null)
{
dynaXml.rp("hivesettings")
.Catalog.b.DatabaseName(cluster.HiveMetastore.Database)
.Password(cluster.HiveMetastore.Password)
.Server(cluster.HiveMetastore.Server)
.Username(cluster.HiveMetastore.User)
.d.End();
}
}
private async Task ValidateCreateClusterSucceeds(ClusterCreateParametersV2 cluster, Action<ClusterDetails> postCreateValidation)
{
IHDInsightCertificateCredential credentials = IntegrationTestBase.GetValidCredentials();
using (var client = ServiceLocator.Instance.Locate<IHDInsightManagementPocoClientFactory>().Create(credentials, GetAbstractionContext(), false))
{
client.CreateContainer(cluster).Wait();
await client.WaitForClusterInConditionOrError(null, cluster.Name, cluster.Location, TimeSpan.FromMinutes(30), HDInsightClient.DefaultPollingInterval, GetAbstractionContext(), this.CreatingStates);
var task = client.ListContainer(cluster.Name);
task.WaitForResult();
var container = task.Result;
Assert.IsNotNull(container);
if (container.Error.IsNotNull())
{
Assert.Fail("The Container was not expected to return an error but returned ({0}) ({1})", container.Error.HttpCode, container.Error.Message);
}
if (postCreateValidation != null)
{
postCreateValidation(container);
}
await client.DeleteContainer(cluster.Name);
await client.WaitForClusterNullOrError(cluster.Name, cluster.Location, TimeSpan.FromMinutes(10), IntegrationTestBase.GetAbstractionContext().CancellationToken);
Assert.IsNull(container.Error);
Assert.IsNull(client.ListContainer(cluster.Name).WaitForResult());
}
}
private async Task ValidateClusterVersion(ClusterCreateParametersV2 cluster)
{
var overrideHandlers = ServiceLocator.Instance.Locate <IHDInsightClusterOverrideManager>().GetHandlers(this.credentials, this.Context, this.IgnoreSslErrors);
// Validates the version for cluster creation
if (!string.IsNullOrEmpty(cluster.Version) && !string.Equals(cluster.Version, DEFAULTHDINSIGHTVERSION, StringComparison.OrdinalIgnoreCase))
{
this.AssertSupportedVersion(overrideHandlers.PayloadConverter.ConvertStringToVersion(cluster.Version));
var availableVersions = await overrideHandlers.VersionFinder.ListAvailableVersions();
if (availableVersions.All(hdinsightVersion => hdinsightVersion.Version != cluster.Version))
{
throw new InvalidOperationException(
string.Format(
"Cannot create a cluster with version '{0}'. Available Versions for your subscription are: {1}",
cluster.Version,
string.Join(",", availableVersions)));
}
// Clusters with OSType.Linux only supported from version 3.2 onwards
var version = new Version(cluster.Version);
if (cluster.OSType == OSType.Linux && version.CompareTo(new Version("3.2")) < 0)
{
throw new NotSupportedException(string.Format("Clusters with OSType {0} are only supported from version 3.2", cluster.OSType));
}
// HBase cluster only supported after version 3.0
if (version.CompareTo(new Version("3.0")) < 0 && cluster.ClusterType == ClusterType.HBase)
{
throw new InvalidOperationException(
string.Format("Cannot create a HBase cluster with version '{0}'. HBase cluster only supported after version 3.0", cluster.Version));
}
// Cluster customization only supported after version 3.0
if (version.CompareTo(new Version("3.0")) < 0 && cluster.ConfigActions != null && cluster.ConfigActions.Count > 0)
{
throw new InvalidOperationException(
string.Format("Cannot create a customized cluster with version '{0}'. Customized clusters only supported after version 3.0", cluster.Version));
}
// Various VM sizes only supported starting with version 3.1
if (version.CompareTo(new Version("3.1")) < 0 && createHasNewVMSizesSpecified(cluster))
{
throw new InvalidOperationException(
string.Format(
"Cannot use various VM sizes with cluster version '{0}'. Custom VM sizes are only supported for cluster versions 3.1 and above.",
cluster.Version));
}
// Spark cluster only supported after version 3.2
if (version.CompareTo(new Version("3.2")) < 0 && cluster.ClusterType == ClusterType.Spark)
{
throw new InvalidOperationException(
string.Format("Cannot create a Spark cluster with version '{0}'. Spark cluster only supported after version 3.2", cluster.Version));
}
}
else
{
cluster.Version = DEFAULTHDINSIGHTVERSION;
}
}
public ClusterDetails CreateCluster(ClusterCreateParametersV2 cluster)
{
return(this.CreateClusterAsync(cluster).WaitForResult());
}
private async Task <ClusterDetails> CreateIaasClusterAsync(ClusterCreateParametersV2 clusterCreateParameters)
{
if (clusterCreateParameters == null)
{
throw new ArgumentNullException("clusterCreateParameters");
}
// Validate cluster creation parameters
clusterCreateParameters.ValidateClusterCreateParameters();
this.LogMessage("Validating Cluster Versions", Severity.Informational, Verbosity.Detailed);
await this.ValidateClusterVersion(clusterCreateParameters);
IHDInsightManagementPocoClient client = this.CreateIaasClustersPocoClient(this.capabilities.Value);
// listen to cluster provisioning events on the POCO client.
client.ClusterProvisioning += this.RaiseClusterProvisioningEvent;
Exception requestException = null;
// Creates a cluster and waits for it to complete
try
{
this.LogMessage("Sending Cluster Create Request", Severity.Informational, Verbosity.Detailed);
await client.CreateContainer(clusterCreateParameters);
}
catch (Exception ex)
{
ex = ex.GetFirstException();
var hlex = ex as HttpLayerException;
var httpEx = ex as HttpRequestException;
var webex = ex as WebException;
if (hlex.IsNotNull() || httpEx.IsNotNull() || webex.IsNotNull())
{
requestException = ex;
if (hlex.IsNotNull())
{
HandleCreateHttpLayerException(clusterCreateParameters, hlex);
}
}
else
{
throw;
}
}
await client.WaitForClusterInConditionOrError(this.HandleClusterWaitNotifyEvent,
clusterCreateParameters.Name,
clusterCreateParameters.Location,
clusterCreateParameters.CreateTimeout,
this.PollingInterval,
this.Context,
ClusterState.Operational,
ClusterState.Running);
// Validates that cluster didn't get on error state
var result = this.currentDetails;
if (result == null)
{
if (requestException != null)
{
throw requestException;
}
throw new HDInsightClusterCreateException("Attempting to return the newly created cluster returned no cluster. The cluster could not be found.");
}
if (result.Error != null)
{
throw new HDInsightClusterCreateException(result);
}
return(result);
}
internal static void ValidateClusterConfiguration(ClusterDetails testCluster, ClusterCreateParametersV2 cluster)
{
var remoteCreds = new BasicAuthCredential()
{
Server = GatewayUriResolver.GetGatewayUri(new Uri(testCluster.ConnectionUrl).Host),
UserName = testCluster.HttpUserName,
Password = testCluster.HttpPassword
};
var configurationAccessor =
ServiceLocator.Instance.Locate<IAzureHDInsightClusterConfigurationAccessorFactory>().Create(remoteCreds);
var coreConfiguration = configurationAccessor.GetCoreServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.CoreConfiguration, coreConfiguration);
var hdfsConfiguration = configurationAccessor.GetHdfsServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.HdfsConfiguration, hdfsConfiguration);
var mapReduceConfiguration = configurationAccessor.GetMapReduceServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.MapReduceConfiguration.ConfigurationCollection, mapReduceConfiguration);
var hiveConfiguration = configurationAccessor.GetHiveServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.HiveConfiguration.ConfigurationCollection, hiveConfiguration);
var oozieConfiguration = configurationAccessor.GetOozieServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.OozieConfiguration.ConfigurationCollection, oozieConfiguration);
}
private void SerializeOozieConfiguration(ClusterCreateParametersV2 cluster, dynamic dynaXml)
{
this.AddConfigurationOptions(dynaXml, cluster.OozieConfiguration.ConfigurationCollection, "ooziesettings");
if (cluster.OozieConfiguration.AdditionalSharedLibraries != null)
{
dynaXml.rp("oozieadditionalsharedlibraries")
.AdditionalSharedLibraries
.b
.AccountName(cluster.OozieConfiguration.AdditionalSharedLibraries.Name)
.BlobContainerName(cluster.OozieConfiguration.AdditionalSharedLibraries.Container)
.Key(cluster.OozieConfiguration.AdditionalSharedLibraries.Key)
.d.End();
}
if (cluster.OozieConfiguration.AdditionalActionExecutorLibraries != null)
{
dynaXml.rp("ooziesharedexecutables")
.AdditionalActionExecutorLibraries
.b
.AccountName(cluster.OozieConfiguration.AdditionalActionExecutorLibraries.Name)
.BlobContainerName(cluster.OozieConfiguration.AdditionalActionExecutorLibraries.Container)
.Key(cluster.OozieConfiguration.AdditionalActionExecutorLibraries.Key)
.d.End();
}
this.SerializeOozieMetastore(cluster, dynaXml);
}
private string CreateClusterRequest_ToInternal(ClusterCreateParametersV2 cluster)
{
dynamic dynaXml = DynaXmlBuilder.Create(false, Formatting.None);
// The RP translates 1 XL into 2 L for SU 4 and up.
// This is done as part of the HA improvement where in the RP would never
// create clusters without 2 nodes for SU 4 release (May '14) and up.
var headnodeCount = cluster.HeadNodeSize.Equals(VmSize.ExtraLarge.ToString()) ? 1 : 2;
dynaXml.xmlns("http://schemas.microsoft.com/windowsazure")
.Resource
.b
.SchemaVersion(SchemaVersion20)
.IntrinsicSettings
.b
.xmlns(May2013)
.ClusterContainer
.b
.Deployment
.b
.AdditionalStorageAccounts
.b
.d
.ClusterPassword(cluster.Password)
.ClusterUsername(cluster.UserName)
.Roles
.b
.ClusterRole
.b
.Count(headnodeCount)
.RoleType(ClusterRoleType.HeadNode)
.VMSize(cluster.HeadNodeSize)
.d
.ClusterRole
.b
.Count(cluster.ClusterSizeInNodes)
.RoleType(ClusterRoleType.DataNode)
.VMSize(cluster.DataNodeSize)
.d
.d
.Version(cluster.Version)
.d
.DeploymentAction(Create)
.ClusterName(cluster.Name)
.Region(cluster.Location)
.StorageAccounts
.b
.BlobContainerReference
.b
.AccountName(cluster.DefaultStorageAccountName)
.BlobContainerName(cluster.DefaultStorageContainer)
.Key(cluster.DefaultStorageAccountKey)
.d
.sp("asv")
.d
.Settings
.b
.Core
.b
.sp("coresettings")
.d
.d
.b
.Hdfs
.b
.sp("hdfssettings")
.d
.d
.b
.MapReduce
.b
.sp("mapreduceconfiguration")
.sp("mapreducecapacityschedulerconfiguration")
.d
.d
.b
.Hive
.b
.sp("hivesettings")
.sp("hiveresources")
.d
.Oozie
.b
.sp("ooziesettings")
.sp("oozieadditionalsharedlibraries")
.sp("ooziesharedexecutables")
.d
.Yarn
.b
.sp("yarnsettings")
.d
.d
.d
.d
.d
.End();
dynaXml.rp("asv");
foreach (var asv in cluster.AdditionalStorageAccounts)
{
this.AddStorageAccount(dynaXml, asv, "deploymentcontainer");
}
this.AddConfigurationOptions(dynaXml, cluster.CoreConfiguration, "coresettings");
this.AddConfigurationOptions(dynaXml, cluster.HdfsConfiguration, "hdfssettings");
this.AddConfigurationOptions(dynaXml, cluster.YarnConfiguration, "yarnsettings");
if (cluster.MapReduceConfiguration != null)
{
this.AddConfigurationOptions(dynaXml, cluster.MapReduceConfiguration.ConfigurationCollection, "mapreduceconfiguration");
this.AddConfigurationOptions(dynaXml, cluster.MapReduceConfiguration.CapacitySchedulerConfigurationCollection, "mapreducecapacityschedulerconfiguration");
}
this.AddConfigurationOptions(dynaXml, cluster.HiveConfiguration.ConfigurationCollection, "hivesettings");
this.SerializeOozieConfiguration(cluster, dynaXml);
this.SerializeHiveConfiguration(cluster, dynaXml);
string xml;
using (var stream = new MemoryStream())
using (var reader = new StreamReader(stream))
{
dynaXml.Save(stream);
stream.Position = 0;
xml = reader.ReadToEnd();
}
return xml;
}
private static void HandleCreateHttpLayerException(ClusterCreateParametersV2 clusterCreateParameters, HttpLayerException e)
{
if (e.RequestContent.Contains(ClusterAlreadyExistsError) && e.RequestStatusCode == HttpStatusCode.BadRequest)
{
throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "Cluster {0} already exists.", clusterCreateParameters.Name));
}
}
private async Task<ClusterDetails> CreateIaasClusterAsync(ClusterCreateParametersV2 clusterCreateParameters)
{
if (clusterCreateParameters == null)
{
throw new ArgumentNullException("clusterCreateParameters");
}
// Validate cluster creation parameters
clusterCreateParameters.ValidateClusterCreateParameters();
this.LogMessage("Validating Cluster Versions", Severity.Informational, Verbosity.Detailed);
await this.ValidateClusterVersion(clusterCreateParameters);
IHDInsightManagementPocoClient client = this.CreateIaasClustersPocoClient(this.capabilities.Value);
// listen to cluster provisioning events on the POCO client.
client.ClusterProvisioning += this.RaiseClusterProvisioningEvent;
Exception requestException = null;
// Creates a cluster and waits for it to complete
try
{
this.LogMessage("Sending Cluster Create Request", Severity.Informational, Verbosity.Detailed);
await client.CreateContainer(clusterCreateParameters);
}
catch (Exception ex)
{
ex = ex.GetFirstException();
var hlex = ex as HttpLayerException;
var httpEx = ex as HttpRequestException;
var webex = ex as WebException;
if (hlex.IsNotNull() || httpEx.IsNotNull() || webex.IsNotNull())
{
requestException = ex;
if (hlex.IsNotNull())
{
HandleCreateHttpLayerException(clusterCreateParameters, hlex);
}
}
else
{
throw;
}
}
await client.WaitForClusterInConditionOrError(this.HandleClusterWaitNotifyEvent,
clusterCreateParameters.Name,
clusterCreateParameters.Location,
clusterCreateParameters.CreateTimeout,
this.PollingInterval,
this.Context,
ClusterState.Operational,
ClusterState.Running);
// Validates that cluster didn't get on error state
var result = this.currentDetails;
if (result == null)
{
if (requestException != null)
{
throw requestException;
}
throw new HDInsightClusterCreateException("Attempting to return the newly created cluster returned no cluster. The cluster could not be found.");
}
if (result.Error != null)
{
throw new HDInsightClusterCreateException(result);
}
return result;
}
/// <inheritdoc />
public async Task<ClusterDetails> CreateClusterAsync(ClusterCreateParametersV2 clusterCreateParameters)
{
if (clusterCreateParameters.OSType == OSType.Linux)
{
return await this.CreateIaasClusterAsync(clusterCreateParameters);
}
else
{
return await this.CreatePaasClusterAsync(clusterCreateParameters);
}
}
public async Task<ClusterDetails> CreateClusterAsync(ClusterCreateParameters clusterCreateParameters)
{
if (clusterCreateParameters == null)
{
throw new ArgumentNullException("clusterCreateParameters");
}
var createParamsV2 = new ClusterCreateParametersV2(clusterCreateParameters);
return await CreateClusterAsync(createParamsV2);
}
private void TestClusterEndToEnd(ClusterCreateParametersV2 cluster, Func<ICollection<ClusterDetails>> getClusters, Func<string, ClusterDetails> getCluster, Func<ClusterCreateParametersV2, ClusterDetails> createCluster, Action<string> deleteCluster)
{
// TODO: DROP ALL THE TABLES IN THE METASTORE TABLES
// Verifies it doesn't exist
var listResult = getClusters();
int matchingContainers = listResult.Count(container => container.Name.Equals(cluster.Name));
Assert.AreEqual(0, matchingContainers);
// Creates the cluster
var result = createCluster(cluster);
Assert.IsNotNull(result);
Assert.IsNotNull(getCluster(cluster.Name));
//validate that we get the storage accounts back from the service.
var defaultStorageAccount = result.DefaultStorageAccount;
Assert.IsNotNull(defaultStorageAccount, cluster.DefaultStorageAccountName);
Assert.AreEqual(cluster.DefaultStorageAccountKey, defaultStorageAccount.Key);
Assert.AreEqual(cluster.DefaultStorageContainer, defaultStorageAccount.Container);
// TODO: USE HADOOP SDK TO LAUNCH A JOB USING BOTH STORAGE ACCOUNTS
// TODO: QUERY SQL METASTORES TO SEE THAT THE DATABASES GOT INITIALIZED
// Deletes the cluster
deleteCluster(cluster.Name);
// Verifies it doesn't exist
Assert.IsNull(getCluster(cluster.Name));
}
private async Task ValidateCreateClusterFailsWithError(ClusterCreateParametersV2 cluster)
{
IHDInsightCertificateCredential credentials = IntegrationTestBase.GetValidCredentials();
var client = ServiceLocator.Instance.Locate<IHDInsightManagementPocoClientFactory>().Create(credentials, GetAbstractionContext(), false);
await client.CreateContainer(cluster);
await client.WaitForClusterNotNull(cluster.Name, cluster.Location, TimeSpan.FromMinutes(10), GetAbstractionContext().CancellationToken);
var result = await client.ListContainer(cluster.Name);
Assert.IsNotNull(result);
Assert.IsNotNull(result.Error);
await client.DeleteContainer(cluster.Name);
await client.WaitForClusterNull(cluster.Name, cluster.Location, TimeSpan.FromMinutes(10), GetAbstractionContext().CancellationToken);
}
/// <inheritdoc />
public string SerializeClusterCreateRequestV3(ClusterCreateParametersV2 cluster)
{
Contracts.May2014.ClusterCreateParameters ccp = null;
if (cluster.ClusterType == ClusterType.HBase)
{
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndHBaseTemplate(cluster);
}
else if (cluster.ClusterType == ClusterType.Storm)
{
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndStormTemplate(cluster);
}
else if (cluster.ClusterType == ClusterType.Spark)
{
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndSparkTemplate(cluster);
}
else if (cluster.ClusterType == ClusterType.Hadoop)
{
ccp = HDInsightClusterRequestGenerator.Create3XClusterFromMapReduceTemplate(cluster);
}
else
{
throw new InvalidDataException("Invalid cluster type");
}
return this.CreateClusterRequest_ToInternalV3(ccp);
}
/// <inheritdoc />
public string SerializeClusterCreateRequest(ClusterCreateParametersV2 cluster)
{
return this.CreateClusterRequest_ToInternal(cluster);
}
public ClusterDetails CreateCluster(ClusterCreateParametersV2 cluster)
{
return this.CreateClusterAsync(cluster).WaitForResult();
}
private void SerializeHiveConfiguration(ClusterCreateParametersV2 cluster, dynamic dynaXml)
{
if (cluster.HiveConfiguration.AdditionalLibraries != null)
{
dynaXml.rp("hiveresources")
.AdditionalLibraries.b.AccountName(cluster.HiveConfiguration.AdditionalLibraries.Name)
.BlobContainerName(cluster.HiveConfiguration.AdditionalLibraries.Container)
.Key(cluster.HiveConfiguration.AdditionalLibraries.Key)
.d.End();
}
if (cluster.HiveMetastore != null)
{
dynaXml.rp("hivesettings")
.Catalog.b.DatabaseName(cluster.HiveMetastore.Database)
.Password(cluster.HiveMetastore.Password)
.Server(cluster.HiveMetastore.Server)
.Username(cluster.HiveMetastore.User)
.d.End();
}
}
public ClusterDetails CreateCluster(ClusterCreateParametersV2 cluster, TimeSpan timeout)
{
return this.CreateClusterAsync(cluster).WaitForResult(timeout);
}
private void SerializeOozieMetastore(ClusterCreateParametersV2 cluster, dynamic dynaXml)
{
if (cluster.OozieMetastore != null)
{
dynaXml.rp("ooziesettings")
.Catalog.b.DatabaseName(cluster.OozieMetastore.Database)
.Password(cluster.OozieMetastore.Password)
.Server(cluster.OozieMetastore.Server)
.Username(cluster.OozieMetastore.User)
.d.End();
}
}
private async Task ValidateClusterVersion(ClusterCreateParametersV2 cluster)
{
var overrideHandlers = ServiceLocator.Instance.Locate<IHDInsightClusterOverrideManager>().GetHandlers(this.credentials, this.Context, this.IgnoreSslErrors);
// Validates the version for cluster creation
if (!string.IsNullOrEmpty(cluster.Version) && !string.Equals(cluster.Version, DEFAULTHDINSIGHTVERSION, StringComparison.OrdinalIgnoreCase))
{
this.AssertSupportedVersion(overrideHandlers.PayloadConverter.ConvertStringToVersion(ClusterVersionUtils.TryGetVersionNumber(cluster.Version)));
var availableVersions = await overrideHandlers.VersionFinder.ListAvailableVersions();
if (availableVersions.All(hdinsightVersion => hdinsightVersion.Version != ClusterVersionUtils.TryGetVersionNumber(cluster.Version)))
{
throw new InvalidOperationException(
string.Format(
"Cannot create a cluster with version '{0}'. Available Versions for your subscription are: {1}",
cluster.Version,
string.Join(",", availableVersions)));
}
// Clusters with OSType.Linux only supported from version 3.2 onwards
var version = new Version(ClusterVersionUtils.TryGetVersionNumber(cluster.Version));
if (cluster.OSType == OSType.Linux && version.CompareTo(new Version("3.2")) < 0)
{
throw new NotSupportedException(string.Format("Clusters with OSType {0} are only supported from version 3.2", cluster.OSType));
}
// HBase cluster only supported after version 3.0
if (version.CompareTo(new Version("3.0")) < 0 && cluster.ClusterType == ClusterType.HBase)
{
throw new InvalidOperationException(
string.Format("Cannot create a HBase cluster with version '{0}'. HBase cluster only supported after version 3.0", cluster.Version));
}
// Cluster customization only supported after version 3.0
if (version.CompareTo(new Version("3.0")) < 0 && cluster.ConfigActions != null && cluster.ConfigActions.Count > 0)
{
throw new InvalidOperationException(
string.Format("Cannot create a customized cluster with version '{0}'. Customized clusters only supported after version 3.0", cluster.Version));
}
// Various VM sizes only supported starting with version 3.1
if (version.CompareTo(new Version("3.1")) < 0 && createHasNewVMSizesSpecified(cluster))
{
throw new InvalidOperationException(
string.Format(
"Cannot use various VM sizes with cluster version '{0}'. Custom VM sizes are only supported for cluster versions 3.1 and above.",
cluster.Version));
}
// Spark cluster only supported after version 3.2
if (version.CompareTo(new Version("3.2")) < 0 && cluster.ClusterType == ClusterType.Spark)
{
throw new InvalidOperationException(
string.Format("Cannot create a Spark cluster with version '{0}'. Spark cluster only supported after version 3.2", cluster.Version));
}
}
else
{
cluster.Version = DEFAULTHDINSIGHTVERSION;
}
}
public ClusterDetails CreateCluster(ClusterCreateParametersV2 cluster)
{
Task<ClusterDetails> createTask = this.CreateClusterAsync(cluster);
createTask.Wait();
return createTask.Result;
}
public Task<ClusterDetails> CreateClusterAsync(ClusterCreateParametersV2 clusterCreateParameters)
{
this.LogMessage("Creating cluster '{0}' in location {1}", clusterCreateParameters.Name, clusterCreateParameters.Location);
LastCreateRequest = clusterCreateParameters;
var clusterDetails = new ClusterDetails();
clusterDetails.Name = clusterCreateParameters.Name;
clusterDetails.HttpPassword = clusterCreateParameters.Password;
clusterDetails.HttpUserName = clusterCreateParameters.UserName;
clusterDetails.Version = clusterCreateParameters.Version;
clusterDetails.Location = clusterCreateParameters.Location;
clusterDetails.State = ClusterState.Running;
clusterDetails.AdditionalStorageAccounts = clusterCreateParameters.AdditionalStorageAccounts;
clusterDetails.DefaultStorageAccount = new WabStorageAccountConfiguration(
clusterCreateParameters.DefaultStorageAccountName,
clusterCreateParameters.DefaultStorageAccountKey,
clusterCreateParameters.DefaultStorageContainer);
Clusters.Add(new SimulatorClusterContainer { Cluster = clusterDetails });
return TaskEx2.FromResult(clusterDetails);
}
private string CreateClusterRequest_ToInternal(ClusterCreateParametersV2 cluster)
{
dynamic dynaXml = DynaXmlBuilder.Create(false, Formatting.None);
// The RP translates 1 XL into 2 L for SU 4 and up.
// This is done as part of the HA improvement where in the RP would never
// create clusters without 2 nodes for SU 4 release (May '14) and up.
var headnodeCount = cluster.HeadNodeSize.Equals(VmSize.ExtraLarge.ToString()) ? 1 : 2;
dynaXml.xmlns("http://schemas.microsoft.com/windowsazure")
.Resource
.b
.SchemaVersion(SchemaVersion20)
.IntrinsicSettings
.b
.xmlns(May2013)
.ClusterContainer
.b
.Deployment
.b
.AdditionalStorageAccounts
.b
.d
.ClusterPassword(cluster.Password)
.ClusterUsername(cluster.UserName)
.Roles
.b
.ClusterRole
.b
.Count(headnodeCount)
.RoleType(ClusterRoleType.HeadNode)
.VMSize(cluster.HeadNodeSize)
.d
.ClusterRole
.b
.Count(cluster.ClusterSizeInNodes)
.RoleType(ClusterRoleType.DataNode)
.VMSize(cluster.DataNodeSize)
.d
.d
.Version(cluster.Version)
.d
.DeploymentAction(Create)
.ClusterName(cluster.Name)
.Region(cluster.Location)
.StorageAccounts
.b
.BlobContainerReference
.b
.AccountName(cluster.DefaultStorageAccountName)
.BlobContainerName(cluster.DefaultStorageContainer)
.Key(cluster.DefaultStorageAccountKey)
.d
.sp("asv")
.d
.Settings
.b
.Core
.b
.sp("coresettings")
.d
.d
.b
.Hdfs
.b
.sp("hdfssettings")
.d
.d
.b
.MapReduce
.b
.sp("mapreduceconfiguration")
.sp("mapreducecapacityschedulerconfiguration")
.d
.d
.b
.Hive
.b
.sp("hivesettings")
.sp("hiveresources")
.d
.Oozie
.b
.sp("ooziesettings")
.sp("oozieadditionalsharedlibraries")
.sp("ooziesharedexecutables")
.d
.Yarn
.b
.sp("yarnsettings")
.d
.d
.d
.d
.d
.End();
dynaXml.rp("asv");
foreach (var asv in cluster.AdditionalStorageAccounts)
{
this.AddStorageAccount(dynaXml, asv, "deploymentcontainer");
}
this.AddConfigurationOptions(dynaXml, cluster.CoreConfiguration, "coresettings");
this.AddConfigurationOptions(dynaXml, cluster.HdfsConfiguration, "hdfssettings");
this.AddConfigurationOptions(dynaXml, cluster.YarnConfiguration, "yarnsettings");
if (cluster.MapReduceConfiguration != null)
{
this.AddConfigurationOptions(dynaXml, cluster.MapReduceConfiguration.ConfigurationCollection, "mapreduceconfiguration");
this.AddConfigurationOptions(dynaXml, cluster.MapReduceConfiguration.CapacitySchedulerConfigurationCollection, "mapreducecapacityschedulerconfiguration");
}
this.AddConfigurationOptions(dynaXml, cluster.HiveConfiguration.ConfigurationCollection, "hivesettings");
this.SerializeOozieConfiguration(cluster, dynaXml);
this.SerializeHiveConfiguration(cluster, dynaXml);
string xml;
using (var stream = new MemoryStream())
using (var reader = new StreamReader(stream))
{
dynaXml.Save(stream);
stream.Position = 0;
xml = reader.ReadToEnd();
}
return(xml);
}
/// <inheritdoc />
public string SerializeClusterCreateRequest(ClusterCreateParametersV2 cluster)
{
return(this.CreateClusterRequest_ToInternal(cluster));
}
public ClusterDetails CreateCluster(ClusterCreateParametersV2 cluster, TimeSpan timeout)
{
return(this.CreateClusterAsync(cluster).WaitForResult(timeout));
}
public async Task CreateContainer(ClusterCreateParametersV2 details)
{
this.LogMessage("Create Cluster Requested", Severity.Informational, Verbosity.Diagnostic);
// Validates that the AzureStorage Configurations are valid and optionally append FQDN suffix to the storage account name
AsvValidationHelper.ValidateAndResolveAsvAccountsAndPrep(details);
// Validates that the config actions' Uris are downloadable.
UriEndpointValidator.ValidateAndResolveConfigActionEndpointUris(details);
var overrideHandlers =
ServiceLocator.Instance.Locate<IHDInsightClusterOverrideManager>()
.GetHandlers(this.credentials, this.Context, this.ignoreSslErrors);
var rdfeCapabilitiesClient =
ServiceLocator.Instance.Locate<IRdfeServiceRestClientFactory>()
.Create(this.credentials, this.Context, this.ignoreSslErrors);
var capabilities = await rdfeCapabilitiesClient.GetResourceProviderProperties();
if (!this.HasClusterCreateCapability(capabilities))
{
throw new InvalidOperationException(string.Format(
"Your subscription cannot create clusters, please contact Support"));
}
// For container resource type, config actions should never be enabled
if (details.ConfigActions != null && details.ConfigActions.Count > 0)
{
throw new InvalidOperationException(string.Format(
"Your subscription cannot create customized clusters, please contact Support"));
}
if (!new[] { "ExtraLarge", "Large" }.Contains(details.HeadNodeSize, StringComparer.OrdinalIgnoreCase)
|| !string.Equals(details.DataNodeSize, "Large", StringComparison.OrdinalIgnoreCase)
|| details.ZookeeperNodeSize.IsNotNullOrEmpty())
{
throw new InvalidOperationException(string.Format(
"Illegal node size specification for container resource. Headnode: [{0}], Datanode: [{1}], Zookeeper: [{2}]",
details.HeadNodeSize, details.DataNodeSize, details.ZookeeperNodeSize));
}
// Validates the region for the cluster creation
var locationClient = ServiceLocator.Instance.Locate<ILocationFinderClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors);
var availableLocations = locationClient.ListAvailableLocations(capabilities);
if (!availableLocations.Contains(details.Location, StringComparer.OrdinalIgnoreCase))
{
throw new InvalidOperationException(string.Format(
"Cannot create a cluster in '{0}'. Available Locations for your subscription are: {1}",
details.Location,
string.Join(",", availableLocations)));
}
// Validates whether the subscription\location needs to be initialized
var registrationClient = ServiceLocator.Instance.Locate<ISubscriptionRegistrationClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors);
await registrationClient.RegisterSubscription();
if (!await registrationClient.ValidateSubscriptionLocation(details.Location))
{
await registrationClient.RegisterSubscriptionLocation(details.Location);
}
// Creates the cluster
var client = ServiceLocator.Instance.Locate<IHDInsightManagementRestClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors);
if (details.ClusterType == ClusterType.HBase || details.ClusterType == ClusterType.Storm || details.ClusterType == ClusterType.Spark)
{
string payload = overrideHandlers.PayloadConverter.SerializeClusterCreateRequestV3(details);
await client.CreateContainer(details.Name, details.Location, payload, 3);
}
else
{
if (!details.VirtualNetworkId.IsNullOrEmpty() || !details.SubnetName.IsNullOrEmpty())
{
throw new InvalidOperationException("Create Hadoop clusters within a virtual network is not permitted.");
}
string payload = overrideHandlers.PayloadConverter.SerializeClusterCreateRequest(details);
await client.CreateContainer(details.Name, details.Location, payload);
}
}
private bool createHasNewVMSizesSpecified(ClusterCreateParametersV2 clusterCreateParameters)
{
const string ExtraLarge = "ExtraLarge";
const string Large = "Large";
if (!new[] {Large, ExtraLarge}.Contains(clusterCreateParameters.HeadNodeSize, StringComparer.OrdinalIgnoreCase))
{
return true;
}
if (!clusterCreateParameters.DataNodeSize.Equals(Large))
{
return true;
}
return clusterCreateParameters.ZookeeperNodeSize != null;
}
private async Task ValidateCreateClusterSucceeds(ClusterCreateParametersV2 cluster)
{
await ValidateCreateClusterSucceeds(cluster, null);
}
public ClusterDetails CreateCluster(ClusterCreateParametersV2 cluster, TimeSpan timeout)
{
return this.CreateCluster(cluster);
}
public static ClusterDetails CreateClusterIfNotExists(ClusterCreateParametersV2 parameters)
{
try
{
LOG.InfoFormat("Checking if the HDInsight cluster exists - SubscriptionId: {0}, Name: {1}",
AppConfig.SubscriptionId, parameters.Name);
var cluster = Client.GetCluster(parameters.Name);
if (cluster == null)
{
LOG.InfoFormat("Submitting a new HDInsight cluster create request - - SubscriptionId: {0}, Name: {1}",
AppConfig.SubscriptionId, parameters.Name);
cluster = Client.CreateCluster(parameters);
}
else
{
LOG.InfoFormat("Skipping creation, found an existing HDInsight Cluster with same name. Name: {0}", parameters.Name);
}
if (cluster.State != ClusterState.Operational && cluster.State != ClusterState.Running)
{
throw new ApplicationException(
String.Format("HDInsight Cluster is not in a operational or running state. Name: {0}, State: {1}",
cluster.Name, cluster.StateString));
}
LOG.InfoFormat("HDInsight cluster created. Details:\r\n{0}", ClusterDetailsAsString(cluster));
return cluster;
}
catch (Exception ex)
{
LOG.Error(
String.Format("Failed to create cluster - SubscriptionId: {0}, Name: {1}", AppConfig.SubscriptionId, parameters.Name),
ex);
throw;
}
}
public Task CreateContainer(ClusterCreateParametersV2 details)
{
this.LastCreateRequest = details;
return underlying.CreateContainer(details);
}
public static ClusterDetails CreateClusterIfNotExists(ClusterType clusterType)
{
var name = AppConfig.AzureResourceName + clusterType.ToString().ToLowerInvariant();
LOG.Info(String.Format("Creating Cluster: Name = {0}, Type = {1}", name, clusterType));
var parameters = new ClusterCreateParametersV2()
{
Name = name,
ClusterSizeInNodes = HDInsightClusterSize,
Location = AppConfig.AzureResourceLocation,
ClusterType = clusterType,
HeadNodeSize = "Large",
DataNodeSize = "Large",
ZookeeperNodeSize = "Medium",
UserName = AppConfig.AzureResourceUsername,
Password = AppConfig.AzureResourcePassword,
DefaultStorageAccountName = AzureStorageHelper.Name,
DefaultStorageAccountKey = AzureStorageHelper.PrimaryKey,
DefaultStorageContainer = AppConfig.AzureResourceName,
Version = "3.2",
//VirtualNetworkId = VirtualNetworkHelper.VNetId,
//SubnetName = VirtualNetworkHelper.SubnetName,
};
return CreateClusterIfNotExists(parameters);
}