private void SerializeOozieConfiguration(ClusterCreateParameters cluster, dynamic dynaXml)
{
this.AddConfigurationOptions(dynaXml, cluster.OozieConfiguration.ConfigurationCollection, "ooziesettings");
if (cluster.OozieConfiguration.AdditionalSharedLibraries != null)
{
dynaXml.rp("oozieadditionalsharedlibraries")
.AdditionalSharedLibraries
.b
.AccountName(cluster.OozieConfiguration.AdditionalSharedLibraries.Name)
.BlobContainerName(cluster.OozieConfiguration.AdditionalSharedLibraries.Container)
.Key(cluster.OozieConfiguration.AdditionalSharedLibraries.Key)
.d.End();
}
if (cluster.OozieConfiguration.AdditionalActionExecutorLibraries != null)
{
dynaXml.rp("ooziesharedexecutables")
.AdditionalActionExecutorLibraries
.b
.AccountName(cluster.OozieConfiguration.AdditionalActionExecutorLibraries.Name)
.BlobContainerName(cluster.OozieConfiguration.AdditionalActionExecutorLibraries.Container)
.Key(cluster.OozieConfiguration.AdditionalActionExecutorLibraries.Key)
.d.End();
}
this.SerializeOozieMetastore(cluster, dynaXml);
}
private static void HandleCreateHttpLayerException(ClusterCreateParameters clusterCreateParameters, HttpLayerException e)
{
if (e.RequestContent.Contains(ClusterAlreadyExistsError) && e.RequestStatusCode == HttpStatusCode.BadRequest)
{
throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "Cluster {0} already exists.", clusterCreateParameters.Name));
}
}
public async Task <ClusterDetails> CreateClusterAsync(ClusterCreateParameters clusterCreateParameters)
{
if (clusterCreateParameters == null)
{
throw new ArgumentNullException("clusterCreateParameters");
}
var createParamsV2 = new ClusterCreateParametersV2(clusterCreateParameters);
return(await CreateClusterAsync(createParamsV2));
}
private void SerializeOozieMetastore(ClusterCreateParameters cluster, dynamic dynaXml)
{
if (cluster.OozieMetastore != null)
{
dynaXml.rp("ooziesettings")
.Catalog.b.DatabaseName(cluster.OozieMetastore.Database)
.Password(cluster.OozieMetastore.Password)
.Server(cluster.OozieMetastore.Server)
.Username(cluster.OozieMetastore.User)
.d.End();
}
}
public ClusterCreateParametersV2(ClusterCreateParameters versionOneParams) : this()
{
this.Name = versionOneParams.Name;
this.Location = versionOneParams.Location;
this.DefaultStorageAccountName = versionOneParams.DefaultStorageAccountName;
this.DefaultStorageAccountKey = versionOneParams.DefaultStorageAccountKey;
this.DefaultStorageContainer = versionOneParams.DefaultStorageContainer;
this.UserName = versionOneParams.UserName;
this.Password = versionOneParams.Password;
this.ClusterSizeInNodes = versionOneParams.ClusterSizeInNodes;
this.Version = versionOneParams.Version;
//headnode can be default, setting real value in CCPV2
this.HeadNodeSize = versionOneParams.HeadNodeSize != NodeVMSize.Default
? versionOneParams.HeadNodeSize.ToString()
: NodeVMSize.Large.ToString();
this.AdditionalStorageAccounts = versionOneParams.AdditionalStorageAccounts;
this.ConfigActions = versionOneParams.ConfigActions;
this.OozieMetastore = versionOneParams.OozieMetastore;
this.HiveMetastore = versionOneParams.HiveMetastore;
this.CoreConfiguration = versionOneParams.CoreConfiguration;
this.HdfsConfiguration = versionOneParams.HdfsConfiguration;
this.MapReduceConfiguration = versionOneParams.MapReduceConfiguration;
this.HiveConfiguration = versionOneParams.HiveConfiguration;
this.OozieConfiguration = versionOneParams.OozieConfiguration;
this.YarnConfiguration = versionOneParams.YarnConfiguration;
this.HBaseConfiguration = versionOneParams.HBaseConfiguration;
this.StormConfiguration = versionOneParams.StormConfiguration;
this.ClusterType = versionOneParams.ClusterType;
this.CreateTimeout = versionOneParams.CreateTimeout;
this.VirtualNetworkId = versionOneParams.VirtualNetworkId;
this.SubnetName = versionOneParams.SubnetName;
//new parameters in version 2 (default values)
this.DataNodeSize = VmSize.Large.ToString();
this.ZookeeperNodeSize = null;
// By default create Windows clusters
this.OSType = HDInsight.OSType.Windows;
}
private void SerializeHiveConfiguration(ClusterCreateParameters cluster, dynamic dynaXml)
{
if (cluster.HiveConfiguration.AdditionalLibraries != null)
{
dynaXml.rp("hiveresources")
.AdditionalLibraries.b.AccountName(cluster.HiveConfiguration.AdditionalLibraries.Name)
.BlobContainerName(cluster.HiveConfiguration.AdditionalLibraries.Container)
.Key(cluster.HiveConfiguration.AdditionalLibraries.Key)
.d.End();
}
if (cluster.HiveMetastore != null)
{
dynaXml.rp("hivesettings")
.Catalog.b.DatabaseName(cluster.HiveMetastore.Database)
.Password(cluster.HiveMetastore.Password)
.Server(cluster.HiveMetastore.Server)
.Username(cluster.HiveMetastore.User)
.d.End();
}
}
/// <inheritdoc />
public string SerializeClusterCreateRequestV3(ClusterCreateParameters cluster)
{
Contracts.May2014.ClusterCreateParameters ccp = null;
if (cluster.ClusterType == ClusterType.HBase)
{
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndHBaseTemplate(cluster);
}
else if (cluster.ClusterType == ClusterType.Storm)
{
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndStormTemplate(cluster);
}
else if (cluster.ClusterType == ClusterType.Hadoop)
{
ccp = HDInsightClusterRequestGenerator.Create3XClusterFromMapReduceTemplate(cluster);
}
else
{
throw new InvalidDataException("Invalid cluster type");
}
return(this.CreateClusterRequest_ToInternalV3(ccp));
}
private async Task ValidateClusterVersion(ClusterCreateParameters cluster)
{
var overrideHandlers = ServiceLocator.Instance.Locate <IHDInsightClusterOverrideManager>().GetHandlers(this.credentials, this.Context, this.IgnoreSslErrors);
// Validates the version for cluster creation
if (!string.IsNullOrEmpty(cluster.Version) && !string.Equals(cluster.Version, DEFAULTHDINSIGHTVERSION, StringComparison.OrdinalIgnoreCase))
{
this.AssertSupportedVersion(overrideHandlers.PayloadConverter.ConvertStringToVersion(cluster.Version));
var availableVersions = await overrideHandlers.VersionFinder.ListAvailableVersions();
if (availableVersions.All(hdinsightVersion => hdinsightVersion.Version != cluster.Version))
{
throw new InvalidOperationException(
string.Format(
"Cannot create a cluster with version '{0}'. Available Versions for your subscription are: {1}",
cluster.Version,
string.Join(",", availableVersions)));
}
// HBase cluster only supported after version 3.0
var version = new Version(cluster.Version);
if (version.CompareTo(new Version("3.0")) < 0 && cluster.ClusterType == ClusterType.HBase)
{
throw new InvalidOperationException(
string.Format("Cannot create a HBase cluster with version '{0}'. HBase cluster only supported after version 3.0", cluster.Version));
}
// Cluster customization only supported after version 3.0
if (version.CompareTo(new Version("3.0")) < 0 && cluster.ConfigActions != null && cluster.ConfigActions.Count > 0)
{
throw new InvalidOperationException(
string.Format("Cannot create a customized cluster with version '{0}'. Customized clusters only supported after version 3.0", cluster.Version));
}
}
else
{
cluster.Version = DEFAULTHDINSIGHTVERSION;
}
}
private async Task ValidateClusterVersion(ClusterCreateParameters cluster)
{
var overrideHandlers = ServiceLocator.Instance.Locate<IHDInsightClusterOverrideManager>().GetHandlers(this.credentials, this.Context, this.IgnoreSslErrors);
// Validates the version for cluster creation
if (!string.IsNullOrEmpty(cluster.Version) && !string.Equals(cluster.Version, DEFAULTHDINSIGHTVERSION, StringComparison.OrdinalIgnoreCase))
{
this.AssertSupportedVersion(overrideHandlers.PayloadConverter.ConvertStringToVersion(cluster.Version));
var availableVersions = await overrideHandlers.VersionFinder.ListAvailableVersions();
if (availableVersions.All(hdinsightVersion => hdinsightVersion.Version != cluster.Version))
{
throw new InvalidOperationException(
string.Format(
"Cannot create a cluster with version '{0}'. Available Versions for your subscription are: {1}",
cluster.Version,
string.Join(",", availableVersions)));
}
// HBase cluster only supported after version 3.0
var version = new Version(cluster.Version);
if (version.CompareTo(new Version("3.0")) < 0 && cluster.ClusterType == ClusterType.HBase)
{
throw new InvalidOperationException(
string.Format("Cannot create a HBase cluster with version '{0}'. HBase cluster only supported after version 3.0", cluster.Version));
}
// Cluster customization only supported after version 3.0
if (version.CompareTo(new Version("3.0")) < 0 && cluster.ConfigActions != null && cluster.ConfigActions.Count > 0)
{
throw new InvalidOperationException(
string.Format("Cannot create a customized cluster with version '{0}'. Customized clusters only supported after version 3.0", cluster.Version));
}
}
else
{
cluster.Version = DEFAULTHDINSIGHTVERSION;
}
}
public Task<ClusterDetails> CreateClusterAsync(ClusterCreateParameters clusterCreateParameters)
{
return CreateClusterAsync(new ClusterCreateParametersV2(clusterCreateParameters));
}
public ClusterDetails CreateCluster(ClusterCreateParameters cluster)
{
Task<ClusterDetails> createTask = this.CreateClusterAsync(
new ClusterCreateParametersV2( cluster));
createTask.Wait();
return createTask.Result;
}
private string CreateClusterRequest_ToInternal(ClusterCreateParameters cluster)
{
dynamic dynaXml = DynaXmlBuilder.Create(false, Formatting.None);
// The RP translates 1 XL into 2 L for SU 4 and up.
// This is done as part of the HA improvement where in the RP would never
// create clusters without 2 nodes for SU 4 release (May '14) and up.
var headnodeCount = cluster.HeadNodeSize == NodeVMSize.Default ? 1 : 2;
dynaXml.xmlns("http://schemas.microsoft.com/windowsazure")
.Resource
.b
.SchemaVersion(SchemaVersion20)
.IntrinsicSettings
.b
.xmlns(May2013)
.ClusterContainer
.b
.Deployment
.b
.AdditionalStorageAccounts
.b
.d
.ClusterPassword(cluster.Password)
.ClusterUsername(cluster.UserName)
.Roles
.b
.ClusterRole
.b
.Count(headnodeCount)
.RoleType(ClusterRoleType.HeadNode)
.VMSize(cluster.HeadNodeSize.ToNodeVMSize())
.d
.ClusterRole
.b
.Count(cluster.ClusterSizeInNodes)
.RoleType(ClusterRoleType.DataNode)
.VMSize(NodeVMSizeInternal.Large)
.d
.d
.Version(cluster.Version)
.d
.DeploymentAction(Create)
.ClusterName(cluster.Name)
.Region(cluster.Location)
.StorageAccounts
.b
.BlobContainerReference
.b
.AccountName(cluster.DefaultStorageAccountName)
.BlobContainerName(cluster.DefaultStorageContainer)
.Key(cluster.DefaultStorageAccountKey)
.d
.sp("asv")
.d
.Settings
.b
.Core
.b
.sp("coresettings")
.d
.d
.b
.Hdfs
.b
.sp("hdfssettings")
.d
.d
.b
.MapReduce
.b
.sp("mapreduceconfiguration")
.sp("mapreducecapacityschedulerconfiguration")
.d
.d
.b
.Hive
.b
.sp("hivesettings")
.sp("hiveresources")
.d
.Oozie
.b
.sp("ooziesettings")
.sp("oozieadditionalsharedlibraries")
.sp("ooziesharedexecutables")
.d
.Yarn
.b
.sp("yarnsettings")
.d
.d
.d
.d
.d
.End();
dynaXml.rp("asv");
foreach (var asv in cluster.AdditionalStorageAccounts)
{
this.AddStorageAccount(dynaXml, asv, "deploymentcontainer");
}
this.AddConfigurationOptions(dynaXml, cluster.CoreConfiguration, "coresettings");
this.AddConfigurationOptions(dynaXml, cluster.HdfsConfiguration, "hdfssettings");
this.AddConfigurationOptions(dynaXml, cluster.YarnConfiguration, "yarnsettings");
if (cluster.MapReduceConfiguration != null)
{
this.AddConfigurationOptions(dynaXml, cluster.MapReduceConfiguration.ConfigurationCollection, "mapreduceconfiguration");
this.AddConfigurationOptions(dynaXml, cluster.MapReduceConfiguration.CapacitySchedulerConfigurationCollection, "mapreducecapacityschedulerconfiguration");
}
this.AddConfigurationOptions(dynaXml, cluster.HiveConfiguration.ConfigurationCollection, "hivesettings");
this.SerializeOozieConfiguration(cluster, dynaXml);
this.SerializeHiveConfiguration(cluster, dynaXml);
string xml;
using (var stream = new MemoryStream())
using (var reader = new StreamReader(stream))
{
dynaXml.Save(stream);
stream.Position = 0;
xml = reader.ReadToEnd();
}
return(xml);
}
private void SerializeOozieConfiguration(ClusterCreateParameters cluster, dynamic dynaXml)
{
this.AddConfigurationOptions(dynaXml, cluster.OozieConfiguration.ConfigurationCollection, "ooziesettings");
if (cluster.OozieConfiguration.AdditionalSharedLibraries != null)
{
dynaXml.rp("oozieadditionalsharedlibraries")
.AdditionalSharedLibraries
.b
.AccountName(cluster.OozieConfiguration.AdditionalSharedLibraries.Name)
.BlobContainerName(cluster.OozieConfiguration.AdditionalSharedLibraries.Container)
.Key(cluster.OozieConfiguration.AdditionalSharedLibraries.Key)
.d.End();
}
if (cluster.OozieConfiguration.AdditionalActionExecutorLibraries != null)
{
dynaXml.rp("ooziesharedexecutables")
.AdditionalActionExecutorLibraries
.b
.AccountName(cluster.OozieConfiguration.AdditionalActionExecutorLibraries.Name)
.BlobContainerName(cluster.OozieConfiguration.AdditionalActionExecutorLibraries.Container)
.Key(cluster.OozieConfiguration.AdditionalActionExecutorLibraries.Key)
.d.End();
}
this.SerializeOozieMetastore(cluster, dynaXml);
}
private string CreateClusterRequest_ToInternal(ClusterCreateParameters cluster)
{
dynamic dynaXml = DynaXmlBuilder.Create(false, Formatting.None);
// The RP translates 1 XL into 2 L for SU 4 and up.
// This is done as part of the HA improvement where in the RP would never
// create clusters without 2 nodes for SU 4 release (May '14) and up.
var headnodeCount = cluster.HeadNodeSize == NodeVMSize.Default ? 1 : 2;
dynaXml.xmlns("http://schemas.microsoft.com/windowsazure")
.Resource
.b
.SchemaVersion(SchemaVersion20)
.IntrinsicSettings
.b
.xmlns(May2013)
.ClusterContainer
.b
.Deployment
.b
.AdditionalStorageAccounts
.b
.d
.ClusterPassword(cluster.Password)
.ClusterUsername(cluster.UserName)
.Roles
.b
.ClusterRole
.b
.Count(headnodeCount)
.RoleType(ClusterRoleType.HeadNode)
.VMSize(cluster.HeadNodeSize.ToNodeVMSize())
.d
.ClusterRole
.b
.Count(cluster.ClusterSizeInNodes)
.RoleType(ClusterRoleType.DataNode)
.VMSize(NodeVMSizeInternal.Large)
.d
.d
.Version(cluster.Version)
.d
.DeploymentAction(Create)
.ClusterName(cluster.Name)
.Region(cluster.Location)
.StorageAccounts
.b
.BlobContainerReference
.b
.AccountName(cluster.DefaultStorageAccountName)
.BlobContainerName(cluster.DefaultStorageContainer)
.Key(cluster.DefaultStorageAccountKey)
.d
.sp("asv")
.d
.Settings
.b
.Core
.b
.sp("coresettings")
.d
.d
.b
.Hdfs
.b
.sp("hdfssettings")
.d
.d
.b
.MapReduce
.b
.sp("mapreduceconfiguration")
.sp("mapreducecapacityschedulerconfiguration")
.d
.d
.b
.Hive
.b
.sp("hivesettings")
.sp("hiveresources")
.d
.Oozie
.b
.sp("ooziesettings")
.sp("oozieadditionalsharedlibraries")
.sp("ooziesharedexecutables")
.d
.Yarn
.b
.sp("yarnsettings")
.d
.d
.d
.d
.d
.End();
dynaXml.rp("asv");
foreach (var asv in cluster.AdditionalStorageAccounts)
{
this.AddStorageAccount(dynaXml, asv, "deploymentcontainer");
}
this.AddConfigurationOptions(dynaXml, cluster.CoreConfiguration, "coresettings");
this.AddConfigurationOptions(dynaXml, cluster.HdfsConfiguration, "hdfssettings");
this.AddConfigurationOptions(dynaXml, cluster.YarnConfiguration, "yarnsettings");
if (cluster.MapReduceConfiguration != null)
{
this.AddConfigurationOptions(dynaXml, cluster.MapReduceConfiguration.ConfigurationCollection, "mapreduceconfiguration");
this.AddConfigurationOptions(dynaXml, cluster.MapReduceConfiguration.CapacitySchedulerConfigurationCollection, "mapreducecapacityschedulerconfiguration");
}
this.AddConfigurationOptions(dynaXml, cluster.HiveConfiguration.ConfigurationCollection, "hivesettings");
this.SerializeOozieConfiguration(cluster, dynaXml);
this.SerializeHiveConfiguration(cluster, dynaXml);
string xml;
using (var stream = new MemoryStream())
using (var reader = new StreamReader(stream))
{
dynaXml.Save(stream);
stream.Position = 0;
xml = reader.ReadToEnd();
}
return xml;
}
/// <inheritdoc />
public string SerializeClusterCreateRequestV3(ClusterCreateParameters cluster)
{
Contracts.May2014.ClusterCreateParameters ccp = null;
if (cluster.ClusterType == ClusterType.HBase)
{
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndHBaseTemplate(cluster);
}
else if (cluster.ClusterType == ClusterType.Storm)
{
ccp = HDInsightClusterRequestGenerator.Create3XClusterForMapReduceAndStormTemplate(cluster);
}
else if (cluster.ClusterType == ClusterType.Hadoop)
{
ccp = HDInsightClusterRequestGenerator.Create3XClusterFromMapReduceTemplate(cluster);
}
else
{
throw new InvalidDataException("Invalid cluster type");
}
return this.CreateClusterRequest_ToInternalV3(ccp);
}
/// <inheritdoc />
public ClusterDetails CreateCluster(ClusterCreateParameters cluster)
{
return this.CreateClusterAsync(new ClusterCreateParametersV2(cluster)).WaitForResult();
}
public async Task<ClusterDetails> CreateClusterAsync(ClusterCreateParameters clusterCreateParameters)
{
if (clusterCreateParameters == null)
{
throw new ArgumentNullException("clusterCreateParameters");
}
var createParamsV2 = new ClusterCreateParametersV2(clusterCreateParameters);
return await CreateClusterAsync(createParamsV2);
}
private async Task ValidateCreateClusterSucceeds(ClusterCreateParameters cluster, Action<ClusterDetails> postCreateValidation)
{
IHDInsightCertificateCredential credentials = IntegrationTestBase.GetValidCredentials();
using (var client = ServiceLocator.Instance.Locate<IHDInsightManagementPocoClientFactory>().Create(credentials, GetAbstractionContext(), false))
{
client.CreateContainer(cluster).Wait();
await client.WaitForClusterInConditionOrError(null, cluster.Name, cluster.Location, TimeSpan.FromMinutes(30), HDInsightClient.DefaultPollingInterval, GetAbstractionContext(), this.CreatingStates);
var task = client.ListContainer(cluster.Name);
task.WaitForResult();
var container = task.Result;
Assert.IsNotNull(container);
if (container.Error.IsNotNull())
{
Assert.Fail("The Container was not expected to return an error but returned ({0}) ({1})", container.Error.HttpCode, container.Error.Message);
}
if (postCreateValidation != null)
{
postCreateValidation(container);
}
await client.DeleteContainer(cluster.Name);
await client.WaitForClusterNullOrError(cluster.Name, cluster.Location, TimeSpan.FromMinutes(10), IntegrationTestBase.GetAbstractionContext().CancellationToken);
Assert.IsNull(container.Error);
Assert.IsNull(client.ListContainer(cluster.Name).WaitForResult());
}
}
private async Task ValidateCreateClusterSucceeds(ClusterCreateParameters cluster)
{
await ValidateCreateClusterSucceeds(cluster, null);
}
internal static void ValidateClusterConfiguration(ClusterDetails testCluster, ClusterCreateParameters cluster)
{
var remoteCreds = new BasicAuthCredential()
{
Server = GatewayUriResolver.GetGatewayUri(new Uri(testCluster.ConnectionUrl).Host),
UserName = testCluster.HttpUserName,
Password = testCluster.HttpPassword
};
var configurationAccessor =
ServiceLocator.Instance.Locate<IAzureHDInsightClusterConfigurationAccessorFactory>().Create(remoteCreds);
var coreConfiguration = configurationAccessor.GetCoreServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.CoreConfiguration, coreConfiguration);
var hdfsConfiguration = configurationAccessor.GetHdfsServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.HdfsConfiguration, hdfsConfiguration);
var mapReduceConfiguration = configurationAccessor.GetMapReduceServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.MapReduceConfiguration.ConfigurationCollection, mapReduceConfiguration);
var hiveConfiguration = configurationAccessor.GetHiveServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.HiveConfiguration.ConfigurationCollection, hiveConfiguration);
var oozieConfiguration = configurationAccessor.GetOozieServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.OozieConfiguration.ConfigurationCollection, oozieConfiguration);
}
private async Task ValidateCreateClusterFailsWithError(ClusterCreateParameters cluster)
{
IHDInsightCertificateCredential credentials = IntegrationTestBase.GetValidCredentials();
var client = ServiceLocator.Instance.Locate<IHDInsightManagementPocoClientFactory>().Create(credentials, GetAbstractionContext(), false);
await client.CreateContainer(cluster);
await client.WaitForClusterNotNull(cluster.Name, cluster.Location, TimeSpan.FromMinutes(10), GetAbstractionContext().CancellationToken);
var result = await client.ListContainer(cluster.Name);
Assert.IsNotNull(result);
Assert.IsNotNull(result.Error);
await client.DeleteContainer(cluster.Name);
await client.WaitForClusterNull(cluster.Name, cluster.Location, TimeSpan.FromMinutes(10), GetAbstractionContext().CancellationToken);
}
public ClusterCreateParametersV2(ClusterCreateParameters versionOneParams) : this()
{
this.Name = versionOneParams.Name;
this.Location = versionOneParams.Location;
this.DefaultStorageAccountName = versionOneParams.DefaultStorageAccountName;
this.DefaultStorageAccountKey = versionOneParams.DefaultStorageAccountKey;
this.DefaultStorageContainer = versionOneParams.DefaultStorageContainer;
this.UserName = versionOneParams.UserName;
this.Password = versionOneParams.Password;
this.ClusterSizeInNodes = versionOneParams.ClusterSizeInNodes;
this.Version = versionOneParams.Version;
//headnode can be default, setting real value in CCPV2
this.HeadNodeSize = versionOneParams.HeadNodeSize != NodeVMSize.Default
? versionOneParams.HeadNodeSize.ToString()
: NodeVMSize.Large.ToString();
this.AdditionalStorageAccounts = versionOneParams.AdditionalStorageAccounts;
this.ConfigActions = versionOneParams.ConfigActions;
this.OozieMetastore = versionOneParams.OozieMetastore;
this.HiveMetastore = versionOneParams.HiveMetastore;
this.CoreConfiguration = versionOneParams.CoreConfiguration;
this.HdfsConfiguration = versionOneParams.HdfsConfiguration;
this.MapReduceConfiguration = versionOneParams.MapReduceConfiguration;
this.HiveConfiguration = versionOneParams.HiveConfiguration;
this.OozieConfiguration = versionOneParams.OozieConfiguration;
this.YarnConfiguration = versionOneParams.YarnConfiguration;
this.HBaseConfiguration = versionOneParams.HBaseConfiguration;
this.StormConfiguration = versionOneParams.StormConfiguration;
this.ClusterType = versionOneParams.ClusterType;
this.CreateTimeout = versionOneParams.CreateTimeout;
this.VirtualNetworkId = versionOneParams.VirtualNetworkId;
this.SubnetName = versionOneParams.SubnetName;
//new parameters in version 2 (default values)
this.DataNodeSize = VmSize.Large.ToString();
this.ZookeeperNodeSize = null;
// By default create Windows clusters
this.OSType = HDInsight.OSType.Windows;
}
/// <inheritdoc />
public ClusterDetails CreateCluster(ClusterCreateParameters cluster)
{
return(this.CreateClusterAsync(cluster).WaitForResult());
}
/// <inheritdoc />
public ClusterDetails CreateCluster(ClusterCreateParameters cluster, TimeSpan timeout)
{
return this.CreateClusterAsync(new ClusterCreateParametersV2(cluster)).WaitForResult(timeout);
}
public Task CreateContainer(ClusterCreateParameters details)
{
this.LastCreateRequest = details;
return underlying.CreateContainer(details);
}
/// <inheritdoc />
public string SerializeClusterCreateRequest(ClusterCreateParameters cluster)
{
return this.CreateClusterRequest_ToInternal(cluster);
}
/// <inheritdoc />
public string SerializeClusterCreateRequest(ClusterCreateParameters cluster)
{
return(this.CreateClusterRequest_ToInternal(cluster));
}
private void SerializeHiveConfiguration(ClusterCreateParameters cluster, dynamic dynaXml)
{
if (cluster.HiveConfiguration.AdditionalLibraries != null)
{
dynaXml.rp("hiveresources")
.AdditionalLibraries.b.AccountName(cluster.HiveConfiguration.AdditionalLibraries.Name)
.BlobContainerName(cluster.HiveConfiguration.AdditionalLibraries.Container)
.Key(cluster.HiveConfiguration.AdditionalLibraries.Key)
.d.End();
}
if (cluster.HiveMetastore != null)
{
dynaXml.rp("hivesettings")
.Catalog.b.DatabaseName(cluster.HiveMetastore.Database)
.Password(cluster.HiveMetastore.Password)
.Server(cluster.HiveMetastore.Server)
.Username(cluster.HiveMetastore.User)
.d.End();
}
}
public Task<ClusterDetails> CreateClusterAsync(ClusterCreateParameters clusterCreateParameters)
{
this.LogMessage("Creating cluster '{0}' in location {1}", clusterCreateParameters.Name, clusterCreateParameters.Location);
LastCreateRequest = clusterCreateParameters;
var clusterDetails = new ClusterDetails();
clusterDetails.Name = clusterCreateParameters.Name;
clusterDetails.HttpPassword = clusterCreateParameters.Password;
clusterDetails.HttpUserName = clusterCreateParameters.UserName;
clusterDetails.Version = clusterCreateParameters.Version;
clusterDetails.Location = clusterCreateParameters.Location;
clusterDetails.State = ClusterState.Running;
clusterDetails.AdditionalStorageAccounts = clusterCreateParameters.AdditionalStorageAccounts;
clusterDetails.DefaultStorageAccount = new WabStorageAccountConfiguration(
clusterCreateParameters.DefaultStorageAccountName,
clusterCreateParameters.DefaultStorageAccountKey,
clusterCreateParameters.DefaultStorageContainer);
Clusters.Add(new SimulatorClusterContainer { Cluster = clusterDetails });
return TaskEx2.FromResult(clusterDetails);
}
private void SerializeOozieMetastore(ClusterCreateParameters cluster, dynamic dynaXml)
{
if (cluster.OozieMetastore != null)
{
dynaXml.rp("ooziesettings")
.Catalog.b.DatabaseName(cluster.OozieMetastore.Database)
.Password(cluster.OozieMetastore.Password)
.Server(cluster.OozieMetastore.Server)
.Username(cluster.OozieMetastore.User)
.d.End();
}
}
public async Task CreateContainer(ClusterCreateParameters details)
{
this.LogMessage("Create Cluster Requested", Severity.Informational, Verbosity.Diagnostic);
// Validates that the AzureStorage Configurations are valid and optionally append FQDN suffix to the storage account name
AsvValidationHelper.ValidateAndResolveAsvAccountsAndPrep(details);
// Validates that the config actions' Uris are downloadable.
UriEndpointValidator.ValidateAndResolveConfigActionEndpointUris(details);
var overrideHandlers = ServiceLocator.Instance.Locate<IHDInsightClusterOverrideManager>().GetHandlers(this.credentials, this.Context, this.ignoreSslErrors);
var rdfeCapabilitiesClient =
ServiceLocator.Instance.Locate<IRdfeServiceRestClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors);
var capabilities = await rdfeCapabilitiesClient.GetResourceProviderProperties();
if (!this.HasClusterCreateCapability(capabilities))
{
throw new InvalidOperationException(string.Format(
"Your subscription cannot create clusters, please contact Support"));
}
// For container resource type, config actions should never be enabled
if (details.ConfigActions != null && details.ConfigActions.Count > 0)
{
throw new InvalidOperationException(string.Format(
"Your subscription cannot create customized clusters, please contact Support"));
}
// Validates the region for the cluster creation
var locationClient = ServiceLocator.Instance.Locate<ILocationFinderClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors);
var availableLocations = locationClient.ListAvailableLocations(capabilities);
if (!availableLocations.Contains(details.Location, StringComparer.OrdinalIgnoreCase))
{
throw new InvalidOperationException(string.Format(
"Cannot create a cluster in '{0}'. Available Locations for your subscription are: {1}",
details.Location,
string.Join(",", availableLocations)));
}
// Validates whether the subscription\location needs to be initialized
var registrationClient = ServiceLocator.Instance.Locate<ISubscriptionRegistrationClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors);
await registrationClient.RegisterSubscription();
if (!await registrationClient.ValidateSubscriptionLocation(details.Location))
{
await registrationClient.RegisterSubscriptionLocation(details.Location);
}
// Creates the cluster
var client = ServiceLocator.Instance.Locate<IHDInsightManagementRestClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors);
if (details.ClusterType == ClusterType.HBase || details.ClusterType == ClusterType.Storm)
{
string payload = overrideHandlers.PayloadConverter.SerializeClusterCreateRequestV3(details);
await client.CreateContainer(details.Name, details.Location, payload, 3);
}
else
{
if (!details.VirtualNetworkId.IsNullOrEmpty() || !details.SubnetName.IsNullOrEmpty())
{
throw new InvalidOperationException("Create Hadoop clusters within a virtual network is not permitted.");
}
string payload = overrideHandlers.PayloadConverter.SerializeClusterCreateRequest(details);
await client.CreateContainer(details.Name, details.Location, payload);
}
}
/// <inheritdoc />
public ClusterDetails CreateCluster(ClusterCreateParameters cluster, TimeSpan timeout)
{
return(this.CreateClusterAsync(new ClusterCreateParametersV2(cluster)).WaitForResult(timeout));
}
/// <inheritdoc />
public async Task <ClusterDetails> CreateClusterAsync(ClusterCreateParameters clusterCreateParameters)
{
if (clusterCreateParameters == null)
{
throw new ArgumentNullException("clusterCreateParameters");
}
IHDInsightManagementPocoClient client = null;
if (!this.canUseClustersContract.Value)
{
client = this.CreateContainersPocoClient();
}
else
{
client = this.CreateClustersPocoClient(this.capabilities.Value);
}
this.LogMessage("Validating Cluster Versions", Severity.Informational, Verbosity.Detailed);
await this.ValidateClusterVersion(clusterCreateParameters);
// listen to cluster provisioning events on the POCO client.
client.ClusterProvisioning += this.RaiseClusterProvisioningEvent;
Exception requestException = null;
// Creates a cluster and waits for it to complete
try
{
this.LogMessage("Sending Cluster Create Request", Severity.Informational, Verbosity.Detailed);
await client.CreateContainer(clusterCreateParameters);
}
catch (Exception ex)
{
ex = ex.GetFirstException();
var hlex = ex as HttpLayerException;
var httpEx = ex as HttpRequestException;
var webex = ex as WebException;
if (hlex.IsNotNull() || httpEx.IsNotNull() || webex.IsNotNull())
{
requestException = ex;
if (hlex.IsNotNull())
{
HandleCreateHttpLayerException(clusterCreateParameters, hlex);
}
}
else
{
throw;
}
}
await client.WaitForClusterInConditionOrError(this.HandleClusterWaitNotifyEvent,
clusterCreateParameters.Name,
clusterCreateParameters.Location,
clusterCreateParameters.CreateTimeout,
this.PollingInterval,
this.Context,
ClusterState.Operational,
ClusterState.Running);
// Validates that cluster didn't get on error state
var result = this.currentDetails;
if (result == null)
{
if (requestException != null)
{
throw requestException;
}
throw new HDInsightClusterCreateException("Attempting to return the newly created cluster returned no cluster. The cluster could not be found.");
}
if (result.Error != null)
{
throw new HDInsightClusterCreateException(result);
}
return(result);
}
public ClusterDetails CreateCluster(ClusterCreateParameters cluster, TimeSpan timeout)
{
return this.CreateCluster(cluster);
}
/// <inheritdoc />
public async Task<ClusterDetails> CreateClusterAsync(ClusterCreateParameters clusterCreateParameters)
{
if (clusterCreateParameters == null)
{
throw new ArgumentNullException("clusterCreateParameters");
}
IHDInsightManagementPocoClient client = null;
if (!this.canUseClustersContract.Value)
{
client = this.CreateContainersPocoClient();
}
else
{
client = this.CreateClustersPocoClient(this.capabilities.Value);
}
this.LogMessage("Validating Cluster Versions", Severity.Informational, Verbosity.Detailed);
await this.ValidateClusterVersion(clusterCreateParameters);
// listen to cluster provisioning events on the POCO client.
client.ClusterProvisioning += this.RaiseClusterProvisioningEvent;
Exception requestException = null;
// Creates a cluster and waits for it to complete
try
{
this.LogMessage("Sending Cluster Create Request", Severity.Informational, Verbosity.Detailed);
await client.CreateContainer(clusterCreateParameters);
}
catch (Exception ex)
{
ex = ex.GetFirstException();
var hlex = ex as HttpLayerException;
var httpEx = ex as HttpRequestException;
var webex = ex as WebException;
if (hlex.IsNotNull() || httpEx.IsNotNull() || webex.IsNotNull())
{
requestException = ex;
if (hlex.IsNotNull())
{
HandleCreateHttpLayerException(clusterCreateParameters, hlex);
}
}
else
{
throw;
}
}
await client.WaitForClusterInConditionOrError(this.HandleClusterWaitNotifyEvent,
clusterCreateParameters.Name,
clusterCreateParameters.Location,
clusterCreateParameters.CreateTimeout,
this.PollingInterval,
this.Context,
ClusterState.Operational,
ClusterState.Running);
// Validates that cluster didn't get on error state
var result = this.currentDetails;
if (result == null)
{
if (requestException != null)
{
throw requestException;
}
throw new HDInsightClusterCreateException("Attempting to return the newly created cluster returned no cluster. The cluster could not be found.");
}
if (result.Error != null)
{
throw new HDInsightClusterCreateException(result);
}
return result;
}
private void TestClusterEndToEnd(ClusterCreateParameters cluster, Func<ICollection<ClusterDetails>> getClusters, Func<string, ClusterDetails> getCluster, Func<ClusterCreateParameters, ClusterDetails> createCluster, Action<string> deleteCluster)
{
// TODO: DROP ALL THE TABLES IN THE METASTORE TABLES
// Verifies it doesn't exist
var listResult = getClusters();
int matchingContainers = listResult.Count(container => container.Name.Equals(cluster.Name));
Assert.AreEqual(0, matchingContainers);
// Creates the cluster
var result = createCluster(cluster);
Assert.IsNotNull(result);
Assert.IsNotNull(getCluster(cluster.Name));
//validate that we get the storage accounts back from the service.
var defaultStorageAccount = result.DefaultStorageAccount;
Assert.IsNotNull(defaultStorageAccount, cluster.DefaultStorageAccountName);
Assert.AreEqual(cluster.DefaultStorageAccountKey, defaultStorageAccount.Key);
Assert.AreEqual(cluster.DefaultStorageContainer, defaultStorageAccount.Container);
// TODO: USE HADOOP SDK TO LAUNCH A JOB USING BOTH STORAGE ACCOUNTS
// TODO: QUERY SQL METASTORES TO SEE THAT THE DATABASES GOT INITIALIZED
// Deletes the cluster
deleteCluster(cluster.Name);
// Verifies it doesn't exist
Assert.IsNull(getCluster(cluster.Name));
}
private static void HandleCreateHttpLayerException(ClusterCreateParameters clusterCreateParameters, HttpLayerException e)
{
if (e.RequestContent.Contains(ClusterAlreadyExistsError) && e.RequestStatusCode == HttpStatusCode.BadRequest)
{
throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "Cluster {0} already exists.", clusterCreateParameters.Name));
}
}