private static void ConfigMapReduceComponent(MapReduceComponent mapReduce, HDInsight.ClusterCreateParameters inputs)
{
mapReduce.MapRedConfXmlProperties.AddRange(
inputs.MapReduceConfiguration.ConfigurationCollection.Select(prop => new Property {
Name = prop.Key, Value = prop.Value
}));
mapReduce.CapacitySchedulerConfXmlProperties.AddRange(
inputs.MapReduceConfiguration.CapacitySchedulerConfigurationCollection.Select(
prop => new Property {
Name = prop.Key, Value = prop.Value
}));
mapReduce.DefaultStorageAccountAndContainer = new BlobContainerCredentialBackedResource()
{
AccountDnsName = inputs.DefaultStorageAccountName,
BlobContainerName = inputs.DefaultStorageContainer,
Key = inputs.DefaultStorageAccountKey
};
if (inputs.AdditionalStorageAccounts.Any())
{
mapReduce.AdditionalStorageAccounts.AddRange(
inputs.AdditionalStorageAccounts.Select(
storageAccount =>
new BlobContainerCredentialBackedResource()
{
AccountDnsName = storageAccount.Name,
BlobContainerName = storageAccount.Container,
Key = storageAccount.Key
}));
}
}
internal void CreateClusterWithoutCapability(string dnsName, string location)
{
var clusterCreateParameters = new HDInsight.ClusterCreateParameters
{
Name = dnsName,
DefaultStorageAccountKey = "storageaccountkey",
DefaultStorageAccountName = "teststorage",
ClusterSizeInNodes = 2,
Location = location,
UserName = "******",
Password = "******",
Version = "3.1"
};
var testCluster = CreateClusterFromCreateParameters(clusterCreateParameters);
List <Cluster> clusters;
bool subExists = RootHandlerSimulatorController._clustersAvailable.TryGetValue(TestSubscription, out clusters);
if (subExists)
{
clusters.Add(testCluster);
RootHandlerSimulatorController._clustersAvailable[TestSubscription] = clusters;
}
else
{
RootHandlerSimulatorController._clustersAvailable.Add(
new KeyValuePair <string, List <Cluster> >(TestSubscription, new List <Cluster> {
testCluster
}));
}
}
private static void ConfigVirtualNetwork(ClusterCreateParameters cluster, HDInsight.ClusterCreateParameters inputs)
{
// Check if the virtual network configuration is partially set
if (string.IsNullOrEmpty(inputs.VirtualNetworkId) ^ string.IsNullOrEmpty(inputs.SubnetName))
{
if (inputs.VirtualNetworkId == null)
{
throw new ArgumentException("Subnet name is set however virtual network GUID is not set.");
}
else
{
throw new ArgumentException("Virtual newtork GUID is set however subnet name is not set.");
}
}
// Set virtual network configuration if is provided in the input
if (!string.IsNullOrEmpty(inputs.VirtualNetworkId) && !string.IsNullOrEmpty(inputs.SubnetName))
{
VirtualNetworkConfiguration virtualNetworkConf = new VirtualNetworkConfiguration();
virtualNetworkConf.VirtualNetworkSite = inputs.VirtualNetworkId;
foreach (var role in cluster.ClusterRoleCollection)
{
AddressAssignment aa = new AddressAssignment();
Subnet subnet = new Subnet();
subnet.Name = inputs.SubnetName;
aa.Subnets.Add(subnet);
aa.Role = role;
virtualNetworkConf.AddressAssignments.Add(aa);
}
cluster.VirtualNetworkConfiguration = virtualNetworkConf;
}
}
public async Task CannotCreateCustomizedClusterWithoutCapability()
{
var restClient = ServiceLocator.Instance.Locate <IRdfeClustersResourceRestClientFactory>()
.Create(this.DefaultHandler, this.HdInsightCertCred, this.Context, false, ClustersPocoClient.GetSchemaVersion(Capabilities));
var clustersPocoClient = new ClustersPocoClient(this.HdInsightCertCred, false, this.Context, Capabilities, restClient);
try
{
var clusterCreateParameters = new HDInsight.ClusterCreateParameters
{
Name = "ConfigActionTest",
DefaultStorageAccountKey = IntegrationTestBase.TestCredentials.Environments[0].DefaultStorageAccount.Key,
DefaultStorageAccountName = IntegrationTestBase.TestCredentials.Environments[0].DefaultStorageAccount.Name,
DefaultStorageContainer = "ConfigActionTest",
ClusterSizeInNodes = 2,
Location = "East US",
UserName = "******",
Password = "******",
Version = "3.1"
};
// Add in valid config action.
clusterCreateParameters.ConfigActions.Add(new ScriptAction("TestScriptAction", new ClusterNodeType[] { ClusterNodeType.HeadNode }, new Uri("http://www.microsoft.com"), null));
await clustersPocoClient.CreateContainer(clusterCreateParameters);
}
catch (NotSupportedException ex)
{
Assert.IsNotNull(ex);
}
}
private static void ConfigHiveComponent(HiveComponent hive, HDInsight.ClusterCreateParameters inputs)
{
hive.HiveSiteXmlProperties.AddRange(
inputs.HiveConfiguration.ConfigurationCollection.Select(prop => new Property {
Name = prop.Key, Value = prop.Value
}));
if (inputs.HiveConfiguration.AdditionalLibraries != null)
{
hive.AdditionalLibraries = new BlobContainerCredentialBackedResource()
{
AccountDnsName = inputs.HiveConfiguration.AdditionalLibraries.Name,
BlobContainerName = inputs.HiveConfiguration.AdditionalLibraries.Container,
Key = inputs.HiveConfiguration.AdditionalLibraries.Key
};
}
if (inputs.HiveMetastore != null)
{
hive.Metastore = new SqlAzureDatabaseCredentialBackedResource()
{
SqlServerName = inputs.HiveMetastore.Server,
Credentials = new UsernamePasswordCredential()
{
Username = inputs.HiveMetastore.User, Password = inputs.HiveMetastore.Password
},
DatabaseName = inputs.HiveMetastore.Database
};
}
}
internal void CreateCluster(string dnsName, string location)
{
var clusterCreateParameters = new HDInsight.ClusterCreateParameters
{
Name = dnsName,
DefaultStorageAccountKey = "storageaccountkey",
DefaultStorageAccountName = "teststorage",
ClusterSizeInNodes = 2,
Location = location,
UserName = "******",
Password = "******",
Version = "3.1"
};
var testCluster = CreateClusterFromCreateParameters(clusterCreateParameters);
testCluster.ClusterCapabilities = new List<string> { ClustersPocoClient.ResizeCapabilityEnabled };
List<Cluster> clusters;
bool subExists = RootHandlerSimulatorController._clustersAvailable.TryGetValue(TestSubscription, out clusters);
if (subExists)
{
clusters.Add(testCluster);
RootHandlerSimulatorController._clustersAvailable[TestSubscription] = clusters;
}
else
{
RootHandlerSimulatorController._clustersAvailable.Add(
new KeyValuePair<string, List<Cluster>>(TestSubscription, new List<Cluster> { testCluster }));
}
}
private static void ConfigHBaseComponent(HBaseComponent hbase, HDInsight.ClusterCreateParameters inputs)
{
hbase.HBaseConfXmlProperties.AddRange(
inputs.HBaseConfiguration.ConfigurationCollection.Select(prop => new Property {
Name = prop.Key, Value = prop.Value
}));
if (inputs.HBaseConfiguration.AdditionalLibraries != null)
{
hbase.AdditionalLibraries = new BlobContainerCredentialBackedResource()
{
AccountDnsName = inputs.HBaseConfiguration.AdditionalLibraries.Name,
BlobContainerName = inputs.HBaseConfiguration.AdditionalLibraries.Container,
Key = inputs.HBaseConfiguration.AdditionalLibraries.Key
};
}
}
/// <summary>
/// Generate ClusterCreateParameters object for 1.X cluster with only Hadoop.
/// </summary>
/// <param name="inputs">The inputs.</param>
/// <returns>An instance of the cluster create parameters.</returns>
internal static ClusterCreateParameters Create1XClusterForMapReduceTemplate(HDInsight.ClusterCreateParameters inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
if (inputs.HeadNodeSize == NodeVMSize.Large)
{
throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, "Version 1.X('{0}') clusters can only contain ExtraLarge headnodes.", inputs.Version));
}
var createParameters = Create2XClusterForMapReduceTemplate(inputs);
var headNodeRole = createParameters.ClusterRoleCollection
.Find(role => role.FriendlyName.Equals("HeadNodeRole", StringComparison.OrdinalIgnoreCase));
//We do not support HA clusters for 1.X so we need to set the instance count to 1
headNodeRole.InstanceCount = 1;
headNodeRole.VMSize = VmSize.ExtraLarge;
return(createParameters);
}
private static Cluster CreateClusterFromCreateParameters(HDInsight.ClusterCreateParameters clusterCreateParameters)
{
var clusterCreateParams = HDInsightClusterRequestGenerator.Create3XClusterFromMapReduceTemplate(clusterCreateParameters);
var cluster = new Cluster
{
ClusterRoleCollection = clusterCreateParams.ClusterRoleCollection,
CreatedTime = DateTime.UtcNow,
Error = null,
FullyQualifiedDnsName = clusterCreateParams.DnsName,
State = ClusterState.Running,
UpdatedTime = DateTime.UtcNow,
DnsName = clusterCreateParams.DnsName,
Components = clusterCreateParams.Components,
ExtensionData = clusterCreateParams.ExtensionData,
Location = clusterCreateParams.Location,
Version = clusterCreateParams.Version,
VirtualNetworkConfiguration = clusterCreateParams.VirtualNetworkConfiguration
};
return(cluster);
}
private static HDInsight.ClusterCreateParameters CreateClusterRequest_FromInternalV3(
Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters payloadObject)
{
var cluster = new HDInsight.ClusterCreateParameters
{
Location = payloadObject.Location,
Name = payloadObject.DnsName,
UserName = GetClusterUsernameFromPayloadObject(payloadObject),
Password = GetClusterPasswordFromPayloadObject(payloadObject),
Version = payloadObject.Version,
DefaultStorageAccountName = GetDefaultStorageAccountFromFromPayloadObject(payloadObject).Name,
DefaultStorageAccountKey = GetDefaultStorageAccountFromFromPayloadObject(payloadObject).Key,
DefaultStorageContainer = GetDefaultStorageAccountFromFromPayloadObject(payloadObject).Container,
ClusterSizeInNodes = payloadObject.ClusterRoleCollection.ToList().Single(role => role.FriendlyName == WorkerNodeRoleName).InstanceCount + 1
};
var headNodeRole = payloadObject.ClusterRoleCollection.ToList().Where(role => role.FriendlyName == HeadNodeRoleName).ToList();
if (headNodeRole.Any())
{
if (headNodeRole.First().VMSize == VmSize.Large)
{
cluster.HeadNodeSize = HDInsight.NodeVMSize.Default;
}
else
{
cluster.HeadNodeSize = (HDInsight.NodeVMSize)((int)(headNodeRole.First().VMSize));
}
}
if (payloadObject.VirtualNetworkConfiguration != null)
{
cluster.VirtualNetworkId = payloadObject.VirtualNetworkConfiguration.VirtualNetworkSite;
cluster.SubnetName = payloadObject.VirtualNetworkConfiguration.AddressAssignments.First().Subnets.First().Name;
}
CopyConfigurationForCluster(payloadObject, cluster);
return(cluster);
}
public void InternalValidation_PayloadConverter_SerializationCreateRequestWithHBaseConfiguration_ResourcesV3()
{
var expected = new HDInsight.ClusterCreateParameters
{
UserName = Guid.NewGuid().ToString("N"),
Password = Guid.NewGuid().ToString("N"),
Version = "3.0",
DefaultStorageAccountKey = Guid.NewGuid().ToString("N"),
DefaultStorageAccountName = Guid.NewGuid().ToString("N"),
DefaultStorageContainer = Guid.NewGuid().ToString("N"),
Name = GetRandomClusterName(),
Location = "East US",
ClusterSizeInNodes = new Random().Next(),
ClusterType = ClusterType.HBase
};
expected.HBaseConfiguration.ConfigurationCollection.Add(new KeyValuePair<string, string>("my setting 1", "my value 1"));
expected.HBaseConfiguration.ConfigurationCollection.Add(new KeyValuePair<string, string>("my setting 2", "my value 2"));
expected.HBaseConfiguration.AdditionalLibraries = new WabStorageAccountConfiguration(
Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString());
string payload = new PayloadConverter().SerializeClusterCreateRequestV3(expected);
var actual = ServerSerializer.DeserializeClusterCreateRequestV3(payload);
Assert.IsTrue(Equals(expected, actual));
}
public void InternalValidation_PayloadConverter_SerializationCreateRequestWithHiveConfiguration()
{
var expected = new HDInsight.ClusterCreateParameters
{
UserName = Guid.NewGuid().ToString("N"),
Password = Guid.NewGuid().ToString("N"),
Version = IntegrationTestBase.TestCredentials.WellKnownCluster.Version,
DefaultStorageAccountKey = Guid.NewGuid().ToString("N"),
DefaultStorageAccountName = Guid.NewGuid().ToString("N"),
DefaultStorageContainer = Guid.NewGuid().ToString("N"),
Name = GetRandomClusterName(),
Location = "East US",
ClusterSizeInNodes = new Random().Next()
};
expected.HiveConfiguration.ConfigurationCollection.Add(new KeyValuePair<string, string>("my setting 1", "my value 1"));
expected.HiveConfiguration.ConfigurationCollection.Add(new KeyValuePair<string, string>("my setting 2", "my value 2"));
string payload = new PayloadConverter().SerializeClusterCreateRequest(expected);
var actual = ServerSerializer.DeserializeClusterCreateRequest(payload);
Assert.IsTrue(Equals(expected, actual));
}
public void InternalValidation_PayloadConverter_SerializationCreateRequestWithMetastore_Storm()
{
var expected = new HDInsight.ClusterCreateParameters
{
UserName = Guid.NewGuid().ToString("N"),
Password = Guid.NewGuid().ToString("N"),
Version = "3.0",
DefaultStorageAccountKey = Guid.NewGuid().ToString("N"),
DefaultStorageAccountName = Guid.NewGuid().ToString("N"),
DefaultStorageContainer = Guid.NewGuid().ToString("N"),
Name = GetRandomClusterName(),
Location = "East US",
ClusterSizeInNodes = new Random().Next(),
ClusterType = ClusterType.Storm
};
expected.AdditionalStorageAccounts.Add(new WabStorageAccountConfiguration(Guid.NewGuid().ToString("N"),
Guid.NewGuid().ToString("N")));
expected.AdditionalStorageAccounts.Add(new WabStorageAccountConfiguration(Guid.NewGuid().ToString("N"),
Guid.NewGuid().ToString("N")));
expected.OozieMetastore = new Metastore(Guid.NewGuid().ToString("N"),
Guid.NewGuid().ToString("N"),
Guid.NewGuid().ToString("N"),
Guid.NewGuid().ToString("N"));
expected.HiveMetastore = new Metastore(Guid.NewGuid().ToString("N"),
Guid.NewGuid().ToString("N"),
Guid.NewGuid().ToString("N"),
Guid.NewGuid().ToString("N"));
string payload = new PayloadConverter().SerializeClusterCreateRequestV3(expected);
var actual = ServerSerializer.DeserializeClusterCreateRequestV3(payload);
Assert.IsTrue(Equals(expected, actual));
}
public void InternalValidation_PayloadConverter_SerializationCreateHadoopClusterRequestWithVirtualNetworkConfigurationV3()
{
var expected = new HDInsight.ClusterCreateParameters
{
UserName = Guid.NewGuid().ToString("N"),
Password = Guid.NewGuid().ToString("N"),
DefaultStorageAccountKey = Guid.NewGuid().ToString("N"),
DefaultStorageAccountName = Guid.NewGuid().ToString("N"),
DefaultStorageContainer = Guid.NewGuid().ToString("N"),
Name = GetRandomClusterName(),
Location = "East US",
Version = "3.0",
ClusterSizeInNodes = new Random().Next()
};
expected.AdditionalStorageAccounts.Add(new WabStorageAccountConfiguration(Guid.NewGuid().ToString("N"),
Guid.NewGuid().ToString("N")));
expected.AdditionalStorageAccounts.Add(new WabStorageAccountConfiguration(Guid.NewGuid().ToString("N"),
Guid.NewGuid().ToString("N")));
expected.VirtualNetworkId = Guid.NewGuid().ToString();
expected.SubnetName = "MySubnet";
string payload = new PayloadConverter().SerializeClusterCreateRequestV3(expected);
var actual = ServerSerializer.DeserializeClusterCreateRequestV3(payload);
Assert.IsTrue(Equals(expected, actual));
}
public void InternalValidation_PayloadConverter_SerializationCreateRequest_MayContracts()
{
var cluster1 = new HDInsight.ClusterCreateParameters
{
Name = "bcarlson",
ClusterSizeInNodes = 1,
UserName = "******",
Version = "default",
Password = "******",
Location = "East US"
};
cluster1.DefaultStorageAccountName = "hdicurrenteastus.blob.core.windows.net";
cluster1.DefaultStorageContainer = "newcontainer";
cluster1.DefaultStorageAccountKey = "jKe7cqoU0a9OmDFlwi3DHZLf7JoKwGOU2pV1iZdBKifxwQuDOKwZFyXMJrPSLtGgDV9b7pVKSGz6lbBWcfX2lA==";
var metaStore = new Metastore("lbl44y45cd.bigbean.windowsazure.mscds.com", "newmaytestdb", "bcarlson", "SuperPass1!");
cluster1.HiveMetastore = cluster1.OozieMetastore = metaStore;
string payload = new PayloadConverter().SerializeClusterCreateRequest(cluster1);
var resource = ServerSerializer.DeserializeClusterCreateRequestIntoResource(payload);
Assert.AreEqual(resource.SchemaVersion, "2.0");
}
public void InternalValidation_PayloadConverter_SerializationCreateRequestV3()
{
var cluster1 = new HDInsight.ClusterCreateParameters
{
UserName = Guid.NewGuid().ToString("N"),
Password = Guid.NewGuid().ToString("N"),
DefaultStorageAccountKey = Guid.NewGuid().ToString("N"),
DefaultStorageAccountName = Guid.NewGuid().ToString("N"),
DefaultStorageContainer = Guid.NewGuid().ToString("N"),
Name = GetRandomClusterName(),
Location = "East US",
Version = "3.0",
ClusterSizeInNodes = new Random().Next(),
ClusterType = ClusterType.HBase
};
cluster1.AdditionalStorageAccounts.Add(new WabStorageAccountConfiguration(Guid.NewGuid().ToString("N"),
Guid.NewGuid().ToString("N")));
cluster1.AdditionalStorageAccounts.Add(new WabStorageAccountConfiguration(Guid.NewGuid().ToString("N"),
Guid.NewGuid().ToString("N")));
string payload = new PayloadConverter().SerializeClusterCreateRequestV3(cluster1);
var cluster2 = ServerSerializer.DeserializeClusterCreateRequestV3(payload);
Assert.IsTrue(Equals(cluster1, cluster2));
}
private static void ConfigStormComponent(StormComponent storm, HDInsight.ClusterCreateParameters inputs)
{
storm.StormConfiguration.AddRange(inputs.StormConfiguration.Select(prop => new Property {
Name = prop.Key, Value = prop.Value
}));
}
/// <summary>
/// Generate ClusterCreateParameters object for 3.X cluster with Hadoop and Storm.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create3XClusterForMapReduceAndStormTemplate(HDInsight.ClusterCreateParameters inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var cluster = Create3XClusterFromMapReduceTemplate(inputs);
var masterRole = cluster.Components.OfType <YarnComponent>().Single().ResourceManagerRole;
var workerRole = cluster.Components.OfType <YarnComponent>().Single().NodeManagerRole;
//Add Storm component
StormComponent storm = new StormComponent
{
MasterRole = masterRole,
WorkerRole = workerRole
};
ConfigStormComponent(storm, inputs);
cluster.Components.Add(storm);
return(cluster);
}
private static void CopyConfiguration(ClusterContainer payloadObject, HDInsight.ClusterCreateParameters cluster)
{
if (payloadObject.Settings.Core != null && payloadObject.Settings.Core.Configuration != null)
{
cluster.CoreConfiguration.AddRange(
payloadObject.Settings.Core.Configuration.Select(config => new KeyValuePair <string, string>(config.Name, config.Value)));
}
if (payloadObject.Settings.Yarn != null && payloadObject.Settings.Yarn.Configuration != null)
{
cluster.YarnConfiguration.AddRange(
payloadObject.Settings.Yarn.Configuration.Select(config => new KeyValuePair <string, string>(config.Name, config.Value)));
}
if (payloadObject.Settings.Hive != null)
{
if (payloadObject.Settings.Hive.AdditionalLibraries != null)
{
cluster.HiveConfiguration.AdditionalLibraries =
new WabStorageAccountConfiguration(
payloadObject.Settings.Hive.AdditionalLibraries.AccountName,
payloadObject.Settings.Hive.AdditionalLibraries.Key,
payloadObject.Settings.Hive.AdditionalLibraries.BlobContainerName);
}
if (payloadObject.Settings.Hive.Configuration != null)
{
cluster.HiveConfiguration.ConfigurationCollection.AddRange(
payloadObject.Settings.Hive.Configuration.Select(config => new KeyValuePair <string, string>(config.Name, config.Value)));
}
}
if (payloadObject.Settings.Hdfs != null && payloadObject.Settings.Hdfs.Configuration != null)
{
cluster.HdfsConfiguration.AddRange(
payloadObject.Settings.Hdfs.Configuration.Select(config => new KeyValuePair <string, string>(config.Name, config.Value)));
}
if (payloadObject.Settings.MapReduce != null && payloadObject.Settings.MapReduce.Configuration != null)
{
cluster.MapReduceConfiguration = new HDInsight.MapReduceConfiguration();
if (payloadObject.Settings.MapReduce.Configuration != null)
{
cluster.MapReduceConfiguration.ConfigurationCollection.AddRange(
payloadObject.Settings.MapReduce.Configuration.Select(config => new KeyValuePair <string, string>(config.Name, config.Value)));
}
if (payloadObject.Settings.MapReduce.CapacitySchedulerConfiguration != null)
{
cluster.MapReduceConfiguration.CapacitySchedulerConfigurationCollection.AddRange(
payloadObject.Settings.MapReduce.CapacitySchedulerConfiguration.Select(config => new KeyValuePair <string, string>(config.Name, config.Value)));
}
}
if (payloadObject.Settings.Oozie != null && payloadObject.Settings.Oozie.Configuration != null)
{
if (cluster.OozieConfiguration.ConfigurationCollection != null)
{
cluster.OozieConfiguration.ConfigurationCollection.AddRange(
payloadObject.Settings.Oozie.Configuration.Select(config => new KeyValuePair <string, string>(config.Name, config.Value)));
}
if (payloadObject.Settings.Oozie.AdditionalSharedLibraries != null)
{
cluster.OozieConfiguration.AdditionalSharedLibraries =
new WabStorageAccountConfiguration(
payloadObject.Settings.Oozie.AdditionalSharedLibraries.AccountName,
payloadObject.Settings.Oozie.AdditionalSharedLibraries.Key,
payloadObject.Settings.Oozie.AdditionalSharedLibraries.BlobContainerName);
}
if (payloadObject.Settings.Oozie.AdditionalActionExecutorLibraries != null)
{
cluster.OozieConfiguration.AdditionalActionExecutorLibraries =
new WabStorageAccountConfiguration(
payloadObject.Settings.Oozie.AdditionalActionExecutorLibraries.AccountName,
payloadObject.Settings.Oozie.AdditionalActionExecutorLibraries.Key,
payloadObject.Settings.Oozie.AdditionalActionExecutorLibraries.BlobContainerName);
}
}
}
private static HDInsight.ClusterCreateParameters CreateClusterRequest_FromInternal(ClusterContainer payloadObject)
{
var cluster = new HDInsight.ClusterCreateParameters
{
Location = payloadObject.Region,
Name = payloadObject.ClusterName
};
cluster.UserName = payloadObject.Deployment.ClusterUsername;
cluster.Password = payloadObject.Deployment.ClusterPassword;
cluster.Version = payloadObject.Deployment.Version;
cluster.DefaultStorageAccountName = payloadObject.StorageAccounts[0].AccountName;
cluster.DefaultStorageAccountKey = payloadObject.StorageAccounts[0].Key;
cluster.DefaultStorageContainer = payloadObject.StorageAccounts[0].BlobContainerName;
var headnodeRole = payloadObject.Deployment.Roles.Single(r => r.RoleType == ClusterRoleType.HeadNode);
//if headnode count is 1 and size XL, then we treat it as Default on the server side
if (headnodeRole.VMSize == Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2013.NodeVMSize.ExtraLarge && headnodeRole.Count == 1)
{
cluster.HeadNodeSize = HDInsight.NodeVMSize.Default;
}
else
{
switch (headnodeRole.VMSize)
{
case Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2013.NodeVMSize.ExtraLarge:
cluster.HeadNodeSize = HDInsight.NodeVMSize.ExtraLarge;
break;
case Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2013.NodeVMSize.Large:
cluster.HeadNodeSize = HDInsight.NodeVMSize.Large;
break;
default:
throw new InvalidDataContractException(string.Format("The server returned an unsupported value for head node VM size '{0}", headnodeRole.VMSize));
}
}
foreach (var asv in payloadObject.StorageAccounts.Skip(1))
{
cluster.AdditionalStorageAccounts.Add(new WabStorageAccountConfiguration(asv.AccountName, asv.Key));
}
if (payloadObject.Settings != null)
{
CopyConfiguration(payloadObject, cluster);
if (payloadObject.Settings.Oozie != null)
{
if (payloadObject.Settings.Oozie.Catalog != null)
{
var oozieMetaStore = payloadObject.Settings.Oozie.Catalog;
cluster.OozieMetastore = new Metastore(oozieMetaStore.Server,
oozieMetaStore.DatabaseName,
oozieMetaStore.Username,
oozieMetaStore.Password);
}
}
if (payloadObject.Settings.Hive != null)
{
if (payloadObject.Settings.Hive.Catalog != null)
{
var hiveMetaStore = payloadObject.Settings.Hive.Catalog;
cluster.HiveMetastore = new Metastore(hiveMetaStore.Server,
hiveMetaStore.DatabaseName,
hiveMetaStore.Username,
hiveMetaStore.Password);
}
}
}
cluster.ClusterSizeInNodes = payloadObject.Deployment.Roles.Sum(role => role.Count);
return(cluster);
}
private static void ConfigHdfsComponent(HdfsComponent hdfs, HDInsight.ClusterCreateParameters inputs)
{
hdfs.HdfsSiteXmlProperties.AddRange(inputs.HdfsConfiguration.Select(prop => new Property {
Name = prop.Key, Value = prop.Value
}));
}
/// <summary>
/// Generate ClusterCreateParameters object for 2.X cluster with only Hadoop.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create2XClusterForMapReduceTemplate(HDInsight.ClusterCreateParameters inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var cluster = new ClusterCreateParameters {
DnsName = inputs.Name, Version = inputs.Version
};
var headnodeRole = new ClusterRole
{
FriendlyName = "HeadNodeRole",
InstanceCount = 2,
VMSize = inputs.HeadNodeSize.ToVmSize(),
};
var workernodeRole = new ClusterRole
{
InstanceCount = inputs.ClusterSizeInNodes,
FriendlyName = "WorkerNodeRole",
VMSize = VmSize.Large
};
var zookeeperRole = new ClusterRole
{
InstanceCount = 3,
FriendlyName = "ZKRole",
VMSize = VmSize.Small
};
cluster.ClusterRoleCollection.Add(headnodeRole);
cluster.ClusterRoleCollection.Add(workernodeRole);
cluster.ClusterRoleCollection.Add(zookeeperRole);
var gateway = new GatewayComponent
{
IsEnabled = true,
RestAuthCredential = new UsernamePasswordCredential {
Username = inputs.UserName, Password = inputs.Password
}
};
cluster.Components.Add(gateway);
cluster.Location = inputs.Location;
// Adding MapReduce component
MapReduceComponent mapReduce = new MapReduceComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
ConfigMapReduceComponent(mapReduce, inputs);
cluster.Components.Add(mapReduce);
// Adding Hive component
HiveComponent hive = new HiveComponent {
HeadNodeRole = headnodeRole
};
ConfigHiveComponent(hive, inputs);
cluster.Components.Add(hive);
// Adding config action component if needed
if (inputs.ConfigActions != null && inputs.ConfigActions.Count > 0)
{
CustomActionComponent configAction = new CustomActionComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
AddConfigActionComponent(configAction, inputs, headnodeRole, workernodeRole);
cluster.Components.Add(configAction);
}
// Adding Oozie component
OozieComponent oozie = new OozieComponent {
HeadNodeRole = headnodeRole
};
ConfigOozieComponent(oozie, inputs);
cluster.Components.Add(oozie);
// Adding Hdfs component
HdfsComponent hdfs = new HdfsComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
ConfigHdfsComponent(hdfs, inputs);
cluster.Components.Add(hdfs);
// Adding HadoopCore component
HadoopCoreComponent hadoopCore = new HadoopCoreComponent();
ConfigHadoopCoreComponent(hadoopCore, inputs);
cluster.Components.Add(hadoopCore);
ConfigVirtualNetwork(cluster, inputs);
return(cluster);
}
private static void ConfigHadoopCoreComponent(HadoopCoreComponent hadoopCore, HDInsight.ClusterCreateParameters inputs)
{
hadoopCore.CoreSiteXmlProperties.AddRange(inputs.CoreConfiguration.Select(prop => new Property {
Name = prop.Key, Value = prop.Value
}));
}
private static void AddConfigActionComponent(CustomActionComponent configAction, HDInsight.ClusterCreateParameters inputs, ClusterRole headnodeRole, ClusterRole workernodeRole)
{
configAction.CustomActions = new CustomActionList();
// Converts config action from PS/SDK to wire contract.
foreach (ConfigAction ca in inputs.ConfigActions)
{
CustomAction newConfigAction;
// Based on the config action type defined in SDK, convert them to config action defined in wire contract.
ScriptAction sca = ca as ScriptAction;
if (sca != null)
{
newConfigAction = new ScriptCustomAction
{
Name = ca.Name,
Uri = sca.Uri,
Parameters = sca.Parameters
};
}
else
{
throw new NotSupportedException("No such config action supported.");
}
newConfigAction.ClusterRoleCollection = new ClusterRoleCollection();
// Add in cluster role collection for each config action.
foreach (ClusterNodeType clusterRoleType in ca.ClusterRoleCollection)
{
if (clusterRoleType == ClusterNodeType.HeadNode)
{
newConfigAction.ClusterRoleCollection.Add(headnodeRole);
}
else if (clusterRoleType == ClusterNodeType.DataNode)
{
newConfigAction.ClusterRoleCollection.Add(workernodeRole);
}
else
{
throw new NotSupportedException("No such node type supported.");
}
}
configAction.CustomActions.Add(newConfigAction);
}
}
private static HDInsight.ClusterCreateParameters GetClusterCreateParametersForHeadNodeSize(NodeVMSize headNodeSize)
{
var cluster1 = new HDInsight.ClusterCreateParameters
{
UserName = Guid.NewGuid().ToString("N"),
Password = Guid.NewGuid().ToString("N"),
DefaultStorageAccountKey = Guid.NewGuid().ToString("N"),
DefaultStorageAccountName = Guid.NewGuid().ToString("N"),
DefaultStorageContainer = Guid.NewGuid().ToString("N"),
Name = GetRandomClusterName(),
Location = "East US",
Version = IntegrationTestBase.TestCredentials.WellKnownCluster.Version,
ClusterSizeInNodes = new Random().Next(),
HeadNodeSize = headNodeSize,
};
cluster1.AdditionalStorageAccounts.Add(new WabStorageAccountConfiguration(Guid.NewGuid().ToString("N"), Guid.NewGuid().ToString("N")));
cluster1.AdditionalStorageAccounts.Add(new WabStorageAccountConfiguration(Guid.NewGuid().ToString("N"), Guid.NewGuid().ToString("N")));
return cluster1;
}
private static void ConfigYarnComponent(YarnComponent yarn, HDInsight.ClusterCreateParameters inputs)
{
yarn.Configuration.AddRange(inputs.YarnConfiguration.Select(prop => new Property {
Name = prop.Key, Value = prop.Value
}));
}
public async Task CannotCreateCustomizedClusterWithoutCapability()
{
var restClient = ServiceLocator.Instance.Locate<IRdfeClustersResourceRestClientFactory>()
.Create(this.DefaultHandler, this.HdInsightCertCred, this.Context, false, ClustersPocoClient.GetSchemaVersion(Capabilities));
var clustersPocoClient = new ClustersPocoClient(this.HdInsightCertCred, false, this.Context, Capabilities, restClient);
try
{
var clusterCreateParameters = new HDInsight.ClusterCreateParameters
{
Name = "ConfigActionTest",
DefaultStorageAccountKey = IntegrationTestBase.TestCredentials.Environments[0].DefaultStorageAccount.Key,
DefaultStorageAccountName = IntegrationTestBase.TestCredentials.Environments[0].DefaultStorageAccount.Name,
DefaultStorageContainer = "ConfigActionTest",
ClusterSizeInNodes = 2,
Location = "East US",
UserName = "******",
Password = "******",
Version = "3.1"
};
// Add in valid config action.
clusterCreateParameters.ConfigActions.Add(new ScriptAction("TestScriptAction", new ClusterNodeType[] { ClusterNodeType.HeadNode }, new Uri("http://www.microsoft.com"), null));
await clustersPocoClient.CreateContainer(clusterCreateParameters);
}
catch (NotSupportedException ex)
{
Assert.IsNotNull(ex);
}
}
public void InternalValidation_PayloadConverter_SerializationCreateRequestWithInvalidConfigActionsV3()
{
var testInvalidConfigAction = new HDInsight.ClusterCreateParameters
{
UserName = Guid.NewGuid().ToString("N"),
Password = Guid.NewGuid().ToString("N"),
Version = IntegrationTestBase.TestCredentials.WellKnownCluster.Version,
DefaultStorageAccountKey = Guid.NewGuid().ToString("N"),
DefaultStorageAccountName = Guid.NewGuid().ToString("N"),
DefaultStorageContainer = Guid.NewGuid().ToString("N"),
Name = GetRandomClusterName(),
Location = "East US",
ClusterSizeInNodes = new Random().Next()
};
testInvalidConfigAction.ConfigActions.Add(new ScriptAction("test invalid script action", null, new Uri("http://www.test.com"), "test parameter"));
new PayloadConverter().SerializeClusterCreateRequestV3(testInvalidConfigAction);
}
private static void CopyConfigurationForCluster(
Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters payloadObject, HDInsight.ClusterCreateParameters cluster)
{
var yarn = payloadObject.Components.OfType <YarnComponent>().Single();
var mapreduce = yarn.Applications.OfType <MapReduceApplication>().Single();
var hive = payloadObject.Components.OfType <HiveComponent>().Single();
var oozie = payloadObject.Components.OfType <OozieComponent>().Single();
var hdfs = payloadObject.Components.OfType <HdfsComponent>().Single();
var hadoopCore = payloadObject.Components.OfType <HadoopCoreComponent>().Single();
HBaseComponent hbase = null;
if (payloadObject.Components.OfType <HBaseComponent>().Count() == 1)
{
hbase = payloadObject.Components.OfType <HBaseComponent>().Single();
}
StormComponent storm = null;
if (payloadObject.Components.OfType <StormComponent>().Count() == 1)
{
storm = payloadObject.Components.OfType <StormComponent>().Single();
}
CustomActionComponent configActions = null;
if (payloadObject.Components.OfType <CustomActionComponent>().Count() == 1)
{
configActions = payloadObject.Components.OfType <CustomActionComponent>().Single();
}
if (hadoopCore.CoreSiteXmlProperties.Any())
{
cluster.CoreConfiguration.AddRange(
hadoopCore.CoreSiteXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (hdfs.HdfsSiteXmlProperties.Any())
{
cluster.HdfsConfiguration.AddRange(hdfs.HdfsSiteXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (mapreduce.MapRedSiteXmlProperties.Any())
{
cluster.MapReduceConfiguration.ConfigurationCollection.AddRange(
mapreduce.MapRedSiteXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (mapreduce.CapacitySchedulerConfiguration.Any())
{
cluster.MapReduceConfiguration.CapacitySchedulerConfigurationCollection.AddRange(
mapreduce.CapacitySchedulerConfiguration.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (mapreduce.AdditionalStorageContainers.ToList().Any())
{
cluster.AdditionalStorageAccounts.AddRange(
from BlobContainerCredentialBackedResource tem in mapreduce.AdditionalStorageContainers
select new WabStorageAccountConfiguration(tem.AccountDnsName, tem.Key, tem.BlobContainerName));
}
if (yarn.Configuration.Any())
{
cluster.YarnConfiguration.AddRange(yarn.Configuration.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (hive.HiveSiteXmlProperties.Any())
{
cluster.HiveConfiguration.ConfigurationCollection.AddRange(
hive.HiveSiteXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (hive.AdditionalLibraries != null)
{
cluster.HiveConfiguration.AdditionalLibraries = new WabStorageAccountConfiguration(
hive.AdditionalLibraries.AccountDnsName, hive.AdditionalLibraries.Key, hive.AdditionalLibraries.BlobContainerName);
}
if (!hive.Metastore.ShouldProvisionNew)
{
var metaStore = (SqlAzureDatabaseCredentialBackedResource)hive.Metastore;
cluster.HiveMetastore = new Metastore(
metaStore.SqlServerName, metaStore.DatabaseName, metaStore.Credentials.Username, metaStore.Credentials.Password);
}
if (configActions != null)
{
foreach (var configAction in configActions.CustomActions)
{
ScriptCustomAction sca = configAction as ScriptCustomAction;
if (sca != null)
{
cluster.ConfigActions.Add(new ScriptAction(
sca.Name, ConvertClusterRoleToClusterNodeType(sca), sca.Uri, sca.Parameters));
}
}
}
if (oozie.Configuration.Any())
{
cluster.OozieConfiguration.ConfigurationCollection.AddRange(
oozie.Configuration.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (oozie.AdditionalSharedLibraries != null)
{
cluster.OozieConfiguration.AdditionalSharedLibraries =
new WabStorageAccountConfiguration(
oozie.AdditionalSharedLibraries.AccountDnsName,
oozie.AdditionalSharedLibraries.Key,
oozie.AdditionalSharedLibraries.BlobContainerName);
}
if (oozie.AdditionalActionExecutorLibraries != null)
{
cluster.OozieConfiguration.AdditionalActionExecutorLibraries =
new WabStorageAccountConfiguration(
oozie.AdditionalActionExecutorLibraries.AccountDnsName,
oozie.AdditionalActionExecutorLibraries.Key,
oozie.AdditionalActionExecutorLibraries.BlobContainerName);
}
if (!oozie.Metastore.ShouldProvisionNew)
{
var metaStore = (SqlAzureDatabaseCredentialBackedResource)oozie.Metastore;
cluster.OozieMetastore = new Metastore(
metaStore.SqlServerName, metaStore.DatabaseName, metaStore.Credentials.Username, metaStore.Credentials.Password);
}
if (hbase != null && hbase.HBaseConfXmlProperties.Any())
{
cluster.HBaseConfiguration.ConfigurationCollection.AddRange(
hbase.HBaseConfXmlProperties.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
if (hbase != null && hbase.AdditionalLibraries != null)
{
cluster.HBaseConfiguration.AdditionalLibraries = new WabStorageAccountConfiguration(
hbase.AdditionalLibraries.AccountDnsName, hbase.AdditionalLibraries.Key, hbase.AdditionalLibraries.BlobContainerName);
}
if (storm != null && storm.StormConfiguration.Any())
{
cluster.StormConfiguration.AddRange(
storm.StormConfiguration.Select(prop => new KeyValuePair <string, string>(prop.Name, prop.Value)));
}
}
public void InternalValidation_PayloadConverter_SerializationCreateRequestWithConfigActionsV3()
{
var expected = new HDInsight.ClusterCreateParameters
{
UserName = Guid.NewGuid().ToString("N"),
Password = Guid.NewGuid().ToString("N"),
Version = IntegrationTestBase.TestCredentials.WellKnownCluster.Version,
DefaultStorageAccountKey = Guid.NewGuid().ToString("N"),
DefaultStorageAccountName = Guid.NewGuid().ToString("N"),
DefaultStorageContainer = Guid.NewGuid().ToString("N"),
Name = GetRandomClusterName(),
Location = "East US",
ClusterSizeInNodes = new Random().Next()
};
expected.ConfigActions.Add(new ScriptAction("testconfigaction1", new ClusterNodeType[] { ClusterNodeType.HeadNode }, new Uri("http://www.test1.com"), "test parameter1"));
expected.ConfigActions.Add(new ScriptAction("testconfigaction2", new ClusterNodeType[] { ClusterNodeType.HeadNode, ClusterNodeType.DataNode }, new Uri("http://www.test2.com"), "test parameter2"));
string payload = new PayloadConverter().SerializeClusterCreateRequestV3(expected);
var actual = ServerSerializer.DeserializeClusterCreateRequestV3(payload);
Assert.IsTrue(Equals(expected, actual));
}
/// <summary>
/// Creates the container.
/// </summary>
/// <param name="clusterCreateParameters">The cluster create parameters.</param>
/// <returns>A task.</returns>
public async Task CreateContainer(HDInsight.ClusterCreateParameters clusterCreateParameters)
{
if (clusterCreateParameters == null)
{
throw new ArgumentNullException("clusterCreateParameters");
}
if (string.IsNullOrEmpty(clusterCreateParameters.Name))
{
throw new ArgumentException("ClusterCreateParameters.Name cannot be null or empty", "clusterCreateParameters");
}
if (string.IsNullOrEmpty(clusterCreateParameters.Location))
{
throw new ArgumentException("ClusterCreateParameters.Location cannot be null or empty", "clusterCreateParameters");
}
if (clusterCreateParameters.ClusterSizeInNodes < 1)
{
throw new ArgumentException("clusterCreateParameters.ClusterSizeInNodes must be > 0");
}
try
{
//Validate
AsvValidationHelper.ValidateAndResolveAsvAccountsAndPrep(clusterCreateParameters);
// Validates config action component.
if (clusterCreateParameters.ConfigActions != null && clusterCreateParameters.ConfigActions.Count > 0)
{
this.LogMessage("Validating parameters for config actions.", Severity.Informational, Verbosity.Detailed);
if (!ClustersPocoClient.HasClusterConfigActionCapability(this.capabilities) ||
!ClustersPocoClient.HasCorrectSchemaVersionForConfigAction(this.capabilities))
{
throw new NotSupportedException("Your subscription does not support config actions.");
}
this.LogMessage("Validating URIs for config actions.", Severity.Informational, Verbosity.Detailed);
// Validates that the config actions' Uris are downloadable.
UriEndpointValidator.ValidateAndResolveConfigActionEndpointUris(clusterCreateParameters);
}
var rdfeCapabilitiesClient =
ServiceLocator.Instance.Locate <IRdfeServiceRestClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors);
var capabilities = await rdfeCapabilitiesClient.GetResourceProviderProperties();
// Validates the region for the cluster creation
var locationClient = ServiceLocator.Instance.Locate <ILocationFinderClientFactory>().Create(this.credentials, this.Context, this.ignoreSslErrors);
var availableLocations = locationClient.ListAvailableLocations(capabilities);
if (!availableLocations.Contains(clusterCreateParameters.Location, StringComparer.OrdinalIgnoreCase))
{
throw new InvalidOperationException(string.Format(
"Cannot create a cluster in '{0}'. Available Locations for your subscription are: {1}",
clusterCreateParameters.Location,
string.Join(",", availableLocations)));
}
await this.RegisterSubscriptionIfExistsAsync();
await this.CreateCloudServiceAsyncIfNotExists(clusterCreateParameters.Location);
var wireCreateParameters = PayloadConverterClusters.CreateWireClusterCreateParametersFromUserType(clusterCreateParameters);
var rdfeResourceInputFromWireInput = PayloadConverterClusters.CreateRdfeResourceInputFromWireInput(wireCreateParameters, GetSchemaVersion(this.capabilities));
await
this.rdfeClustersRestClient.CreateCluster(
this.credentials.SubscriptionId.ToString(),
this.GetCloudServiceName(clusterCreateParameters.Location),
this.credentials.DeploymentNamespace,
clusterCreateParameters.Name,
rdfeResourceInputFromWireInput,
this.Context.CancellationToken);
}
catch (InvalidExpectedStatusCodeException iEx)
{
string content = iEx.Response.Content != null?iEx.Response.Content.ReadAsStringAsync().Result : string.Empty;
throw new HttpLayerException(iEx.ReceivedStatusCode, content);
}
}
/// <summary>
/// Generate ClusterCreateParameters object for 3.X cluster with Hadoop and HBase.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create3XClusterForMapReduceAndHBaseTemplate(HDInsight.ClusterCreateParameters inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var cluster = Create3XClusterFromMapReduceTemplate(inputs);
var hbaseMasterRole = cluster.Components.OfType <ZookeeperComponent>().Single().ZookeeperRole;
hbaseMasterRole.VMSize = VmSize.Medium;
//Add HBase component
HBaseComponent hbase = new HBaseComponent
{
MasterServerRole = hbaseMasterRole,
RegionServerRole = cluster.Components.OfType <HdfsComponent>().Single().WorkerNodeRole
};
ConfigHBaseComponent(hbase, inputs);
cluster.Components.Add(hbase);
return(cluster);
}