public override IHttpClientAbstraction Create(string token, HDInsight.IAbstractionContext context, bool ignoreSslErrors)
{
var loc = this.AsyncMock;
if (loc.IsNotNull())
{
return new HttpAbstractionSimulatorClient(this, this.underlying.Create(token, context, ignoreSslErrors), loc);
}
else
{
return new HttpAbstractionSimulatorClient(this, this.underlying.Create(token, context, ignoreSslErrors), null);
}
}
internal HDInsightManagementRestClient(IHDInsightSubscriptionCredentials credentials, HDInsight.IAbstractionContext context, bool ignoreSslErrors)
{
this.context = context;
this.credentials = credentials;
this.ignoreSslErrors = ignoreSslErrors;
if (context.Logger.IsNotNull())
{
this.Logger = context.Logger;
}
else
{
this.Logger = new Logger();
}
}
/// <inheritdoc />
public IHttpClientAbstraction Create(IHDInsightSubscriptionCredentials credentials, HDInsight.IAbstractionContext context, bool ignoreSslErrors)
{
IHDInsightCertificateCredential certCreds = credentials as IHDInsightCertificateCredential;
IHDInsightAccessTokenCredential tokenCreds = credentials as IHDInsightAccessTokenCredential;
if (certCreds != null)
{
return
ServiceLocator.Instance.Locate<IHttpClientAbstractionFactory>()
.Create(certCreds.Certificate, context, ignoreSslErrors);
}
if (tokenCreds != null)
{
return
ServiceLocator.Instance.Locate<IHttpClientAbstractionFactory>()
.Create(tokenCreds.AccessToken, context, ignoreSslErrors);
}
throw new NotSupportedException("Credential Type is not supported");
}
/// <summary>
/// Generate ClusterCreateParameters object for 1.X cluster with only Hadoop.
/// </summary>
/// <param name="inputs">The inputs.</param>
/// <returns>An instance of the cluster create parameters.</returns>
internal static ClusterCreateParameters Create1XClusterForMapReduceTemplate(HDInsight.ClusterCreateParameters inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
if (inputs.HeadNodeSize == NodeVMSize.Large)
{
throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, "Version 1.X('{0}') clusters can only contain ExtraLarge headnodes.", inputs.Version));
}
var createParameters = Create2XClusterForMapReduceTemplate(inputs);
var headNodeRole = createParameters.ClusterRoleCollection
.Find(role => role.FriendlyName.Equals("HeadNodeRole", StringComparison.OrdinalIgnoreCase));
//We do not support HA clusters for 1.X so we need to set the instance count to 1
headNodeRole.InstanceCount = 1;
headNodeRole.VMSize = VmSize.ExtraLarge;
return createParameters;
}
private static void ConfigHdfsComponent(HdfsComponent hdfs, HDInsight.ClusterCreateParameters inputs)
{
hdfs.HdfsSiteXmlProperties.AddRange(inputs.HdfsConfiguration.Select(prop => new Property { Name = prop.Key, Value = prop.Value }));
}
private ClusterErrorStatus ValidateClusterCreation(HDInsight.ClusterCreateParametersV2 cluster)
{
if (!this.ValidateClusterCreationMetadata(cluster.HiveMetastore, cluster.OozieMetastore))
return new ClusterErrorStatus(400, "Invalid metastores", "create");
return null;
}
internal SubscriptionRegistrationClient(IHDInsightSubscriptionCredentials credentials, HDInsight.IAbstractionContext context, bool ignoreSslErrors)
{
this.context = context;
this.credentials = credentials;
this.ignoreSslErrors = ignoreSslErrors;
}
/// <summary>
/// Generate ClusterCreateParameters object for 2.X cluster with only Hadoop.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create2XClusterForMapReduceTemplate(HDInsight.ClusterCreateParameters inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var cluster = new ClusterCreateParameters { DnsName = inputs.Name, Version = inputs.Version };
var headnodeRole = new ClusterRole
{
FriendlyName = "HeadNodeRole",
InstanceCount = 2,
VMSize = inputs.HeadNodeSize.ToVmSize(),
};
var workernodeRole = new ClusterRole
{
InstanceCount = inputs.ClusterSizeInNodes,
FriendlyName = "WorkerNodeRole",
VMSize = VmSize.Large
};
var zookeeperRole = new ClusterRole
{
InstanceCount = 3,
FriendlyName = "ZKRole",
VMSize = VmSize.Small
};
cluster.ClusterRoleCollection.Add(headnodeRole);
cluster.ClusterRoleCollection.Add(workernodeRole);
cluster.ClusterRoleCollection.Add(zookeeperRole);
var gateway = new GatewayComponent
{
IsEnabled = true,
RestAuthCredential = new UsernamePasswordCredential { Username = inputs.UserName, Password = inputs.Password }
};
cluster.Components.Add(gateway);
cluster.Location = inputs.Location;
// Adding MapReduce component
MapReduceComponent mapReduce = new MapReduceComponent { HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole };
ConfigMapReduceComponent(mapReduce, inputs);
cluster.Components.Add(mapReduce);
// Adding Hive component
HiveComponent hive = new HiveComponent { HeadNodeRole = headnodeRole };
ConfigHiveComponent(hive, inputs);
cluster.Components.Add(hive);
// Adding config action component if needed
if (inputs.ConfigActions != null && inputs.ConfigActions.Count > 0)
{
CustomActionComponent configAction = new CustomActionComponent { HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole };
AddConfigActionComponent(configAction, inputs, headnodeRole, workernodeRole);
cluster.Components.Add(configAction);
}
// Adding Oozie component
OozieComponent oozie = new OozieComponent { HeadNodeRole = headnodeRole };
ConfigOozieComponent(oozie, inputs);
cluster.Components.Add(oozie);
// Adding Hdfs component
HdfsComponent hdfs = new HdfsComponent { HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole };
ConfigHdfsComponent(hdfs, inputs);
cluster.Components.Add(hdfs);
// Adding HadoopCore component
HadoopCoreComponent hadoopCore = new HadoopCoreComponent();
ConfigHadoopCoreComponent(hadoopCore, inputs);
cluster.Components.Add(hadoopCore);
ConfigVirtualNetwork(cluster, inputs);
return cluster;
}
private static void ConfigStormComponent(StormComponent storm, HDInsight.ClusterCreateParameters inputs)
{
storm.StormConfiguration.AddRange(inputs.StormConfiguration.Select(prop => new Property { Name = prop.Key, Value = prop.Value }));
}
private static void ConfigYarnComponent(YarnComponent yarn, HDInsight.ClusterCreateParameters inputs)
{
yarn.Configuration.AddRange(inputs.YarnConfiguration.Select(prop => new Property { Name = prop.Key, Value = prop.Value }));
}
private static void ConfigSparkComponent(SparkComponent spark, HDInsight.ClusterCreateParametersV2 inputs)
{
spark.SparkConfiguration.AddRange(inputs.SparkConfiguration.Select(prop => new Property { Name = prop.Key, Value = prop.Value }));
}
private static void AddConfigActionComponent(CustomActionComponent configAction, HDInsight.ClusterCreateParametersV2 inputs, ClusterRole headnodeRole, ClusterRole workernodeRole, ClusterRole zookeperRole)
{
configAction.CustomActions = new CustomActionList();
// Converts config action from PS/SDK to wire contract.
foreach (ConfigAction ca in inputs.ConfigActions)
{
CustomAction newConfigAction;
// Based on the config action type defined in SDK, convert them to config action defined in wire contract.
ScriptAction sca = ca as ScriptAction;
if (sca != null)
{
newConfigAction = new ScriptCustomAction
{
Name = ca.Name,
Uri = sca.Uri,
Parameters = sca.Parameters
};
}
else
{
throw new NotSupportedException("No such config action supported.");
}
newConfigAction.ClusterRoleCollection = new ClusterRoleCollection();
// Add in cluster role collection for each config action.
foreach (ClusterNodeType clusterRoleType in ca.ClusterRoleCollection)
{
if (clusterRoleType == ClusterNodeType.HeadNode)
{
newConfigAction.ClusterRoleCollection.Add(headnodeRole);
}
else if (clusterRoleType == ClusterNodeType.DataNode)
{
newConfigAction.ClusterRoleCollection.Add(workernodeRole);
}
else if (clusterRoleType == ClusterNodeType.ZookeperNode)
{
if (inputs.ClusterType.Equals(ClusterType.HBase) || inputs.ClusterType.Equals(ClusterType.Storm))
{
newConfigAction.ClusterRoleCollection.Add(zookeperRole);
}
else
{
throw new NotSupportedException(string.Format("Customization of zookeper nodes only supported for cluster types {0} and {1}",
ClusterType.HBase.ToString(), ClusterType.Storm.ToString()));
}
}
else
{
throw new NotSupportedException("No such node type supported.");
}
}
configAction.CustomActions.Add(newConfigAction);
}
}
/// <summary>
/// Generate ClusterCreateParameters object for 3.X cluster with only Hadoop.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create3XClusterFromMapReduceTemplate(HDInsight.ClusterCreateParametersV2 inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var remoteDesktopSettings = (string.IsNullOrEmpty(inputs.RdpUsername))
? new RemoteDesktopSettings()
{
IsEnabled = false
}
: new RemoteDesktopSettings()
{
IsEnabled = true,
AuthenticationCredential = new UsernamePasswordCredential()
{
Username = inputs.RdpUsername,
Password = inputs.RdpPassword
},
RemoteAccessExpiry = (DateTime)inputs.RdpAccessExpiry
};
var cluster = new ClusterCreateParameters
{
DnsName = inputs.Name,
Version = inputs.Version,
};
var headnodeRole = new ClusterRole
{
FriendlyName = "HeadNodeRole",
InstanceCount = 2,
VMSizeAsString = inputs.HeadNodeSize,
RemoteDesktopSettings = remoteDesktopSettings
};
var workernodeRole = new ClusterRole
{
InstanceCount = inputs.ClusterSizeInNodes,
FriendlyName = "WorkerNodeRole",
VMSizeAsString = inputs.DataNodeSize,
RemoteDesktopSettings = remoteDesktopSettings
};
var zookeeperRole = new ClusterRole
{
InstanceCount = 3,
FriendlyName = "ZKRole",
VMSizeAsString = inputs.ZookeeperNodeSize ?? VmSize.Small.ToString(),
RemoteDesktopSettings = remoteDesktopSettings
};
cluster.ClusterRoleCollection.Add(headnodeRole);
cluster.ClusterRoleCollection.Add(workernodeRole);
cluster.ClusterRoleCollection.Add(zookeeperRole);
var gateway = new GatewayComponent
{
IsEnabled = true,
RestAuthCredential = new UsernamePasswordCredential { Username = inputs.UserName, Password = inputs.Password }
};
cluster.Components.Add(gateway);
cluster.Location = inputs.Location;
//Add yarn component
YarnComponent yarn = new YarnComponent { ResourceManagerRole = headnodeRole, NodeManagerRole = workernodeRole, };
ConfigYarnComponent(yarn, inputs);
MapReduceApplication mapreduceApp = new MapReduceApplication();
ConfigMapReduceApplication(mapreduceApp, inputs);
yarn.Applications.Add(mapreduceApp);
cluster.Components.Add(yarn);
// Adding Hive component
HiveComponent hive = new HiveComponent { HeadNodeRole = headnodeRole };
ConfigHiveComponent(hive, inputs);
cluster.Components.Add(hive);
// Adding config action component if needed
if (inputs.ConfigActions != null && inputs.ConfigActions.Count > 0)
{
CustomActionComponent configAction = new CustomActionComponent { HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole };
AddConfigActionComponent(configAction, inputs, headnodeRole, workernodeRole, zookeeperRole);
cluster.Components.Add(configAction);
}
// Adding Oozie component
OozieComponent oozie = new OozieComponent { HeadNodeRole = headnodeRole };
ConfigOozieComponent(oozie, inputs);
cluster.Components.Add(oozie);
// Adding Hdfs component
HdfsComponent hdfs = new HdfsComponent { HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole };
ConfigHdfsComponent(hdfs, inputs);
cluster.Components.Add(hdfs);
// Adding HadoopCore component
HadoopCoreComponent hadoopCore = new HadoopCoreComponent();
ConfigHadoopCoreComponent(hadoopCore, inputs);
cluster.Components.Add(hadoopCore);
// Adding Zookeeper component
cluster.Components.Add(new ZookeeperComponent { ZookeeperRole = zookeeperRole });
ConfigVirtualNetwork(cluster, inputs);
return cluster;
}
/// <inheritdoc />
public IHttpClientAbstraction Create(HDInsight.IAbstractionContext context, bool ignoreSslErrors)
{
return ServiceLocator.Instance.Locate<IHttpClientAbstractionFactory>()
.Create(context, ignoreSslErrors);
}
private static void ConfigHadoopCoreComponent(HadoopCoreComponent hadoopCore, HDInsight.ClusterCreateParameters inputs)
{
hadoopCore.CoreSiteXmlProperties.AddRange(inputs.CoreConfiguration.Select(prop => new Property { Name = prop.Key, Value = prop.Value }));
}
private static void ConfigMapReduceApplication(MapReduceApplication mapReduceApp, HDInsight.ClusterCreateParameters inputs)
{
mapReduceApp.MapRedSiteXmlProperties.AddRange(
inputs.MapReduceConfiguration.ConfigurationCollection.Select(prop => new Property { Name = prop.Key, Value = prop.Value }));
mapReduceApp.CapacitySchedulerConfiguration.AddRange(
inputs.MapReduceConfiguration.CapacitySchedulerConfigurationCollection.Select(
prop => new Property { Name = prop.Key, Value = prop.Value }));
mapReduceApp.DefaultStorageAccountAndContainer = new BlobContainerCredentialBackedResource()
{
AccountDnsName = inputs.DefaultStorageAccountName,
BlobContainerName = inputs.DefaultStorageContainer,
Key = inputs.DefaultStorageAccountKey
};
if (inputs.AdditionalStorageAccounts.Any())
{
mapReduceApp.AdditionalStorageContainers.AddRange(
inputs.AdditionalStorageAccounts.Select(
storageAccount =>
new BlobContainerCredentialBackedResource()
{
AccountDnsName = storageAccount.Name,
BlobContainerName = storageAccount.Container,
Key = storageAccount.Key
}));
}
}
private static bool Equals(HDInsight.ClusterCreateParameters expected, HDInsight.ClusterCreateParameters actual)
{
if (expected == null && actual == null)
{
return true;
}
if (expected == null || actual == null)
{
return false;
}
// Compares the properties and fails if there is a mismatch
var comparisonTuples = new List<Tuple<object, object>>
{
new Tuple<object, object>(expected.UserName, actual.UserName),
new Tuple<object, object>(expected.Password, actual.Password),
new Tuple<object, object>(expected.Version, actual.Version),
new Tuple<object, object>(expected.DefaultStorageAccountKey, actual.DefaultStorageAccountKey),
new Tuple<object, object>(expected.DefaultStorageAccountName, actual.DefaultStorageAccountName),
new Tuple<object, object>(expected.DefaultStorageContainer, actual.DefaultStorageContainer),
new Tuple<object, object>(expected.Name, actual.Name),
new Tuple<object, object>(expected.Location, actual.Location),
new Tuple<object, object>(expected.ClusterSizeInNodes + 1, actual.ClusterSizeInNodes),
new Tuple<object, object>(expected.AdditionalStorageAccounts.Count, actual.AdditionalStorageAccounts.Count),
new Tuple<object, object>(expected.HeadNodeSize, actual.HeadNodeSize),
new Tuple<object, object>(expected.VirtualNetworkId, actual.VirtualNetworkId),
new Tuple<object, object>(expected.SubnetName, actual.SubnetName),
};
if (expected.OozieMetastore != null)
{
Assert.IsNotNull(actual.OozieMetastore, "OozieMetaStore");
comparisonTuples.Add(new Tuple<object, object>(expected.OozieMetastore.Server, actual.OozieMetastore.Server));
comparisonTuples.Add(new Tuple<object, object>(expected.OozieMetastore.Database, actual.OozieMetastore.Database));
comparisonTuples.Add(new Tuple<object, object>(expected.OozieMetastore.User, actual.OozieMetastore.User));
comparisonTuples.Add(new Tuple<object, object>(expected.OozieMetastore.Password, actual.OozieMetastore.Password));
}
if (expected.HiveMetastore != null)
{
Assert.IsNotNull(actual.HiveMetastore, "HiveMetastore");
comparisonTuples.Add(new Tuple<object, object>(expected.HiveMetastore.Server, actual.HiveMetastore.Server));
comparisonTuples.Add(new Tuple<object, object>(expected.HiveMetastore.Database, actual.HiveMetastore.Database));
comparisonTuples.Add(new Tuple<object, object>(expected.HiveMetastore.User, actual.HiveMetastore.User));
comparisonTuples.Add(new Tuple<object, object>(expected.HiveMetastore.Password, actual.HiveMetastore.Password));
}
if (!CompareTuples(comparisonTuples))
{
return false;
}
foreach (var storageAccount in expected.AdditionalStorageAccounts)
{
var storageAccountUnderTest = actual.AdditionalStorageAccounts.FirstOrDefault(storage => storage.Name == storageAccount.Name);
Assert.IsNotNull(storageAccountUnderTest, "Storage account '{0}' was not found.", storageAccount.Name);
Assert.AreEqual(storageAccountUnderTest.Key, storageAccountUnderTest.Key);
}
foreach (var configAction in expected.ConfigActions)
{
Assert.IsTrue(actual.ConfigActions.Any(ca => ca.Equals(configAction)));
}
if (expected.OozieConfiguration.AdditionalSharedLibraries != null)
{
Assert.IsNotNull(actual.OozieConfiguration.AdditionalSharedLibraries);
Assert.AreEqual(actual.OozieConfiguration.AdditionalSharedLibraries.Container, actual.OozieConfiguration.AdditionalSharedLibraries.Container);
Assert.AreEqual(actual.OozieConfiguration.AdditionalSharedLibraries.Name, actual.OozieConfiguration.AdditionalSharedLibraries.Name);
Assert.AreEqual(actual.OozieConfiguration.AdditionalSharedLibraries.Key, actual.OozieConfiguration.AdditionalSharedLibraries.Key);
}
if (expected.OozieConfiguration.AdditionalActionExecutorLibraries != null)
{
Assert.IsNotNull(actual.OozieConfiguration.AdditionalActionExecutorLibraries);
Assert.AreEqual(actual.OozieConfiguration.AdditionalActionExecutorLibraries.Container, actual.OozieConfiguration.AdditionalActionExecutorLibraries.Container);
Assert.AreEqual(actual.OozieConfiguration.AdditionalActionExecutorLibraries.Name, actual.OozieConfiguration.AdditionalActionExecutorLibraries.Name);
Assert.AreEqual(actual.OozieConfiguration.AdditionalActionExecutorLibraries.Key, actual.OozieConfiguration.AdditionalActionExecutorLibraries.Key);
}
AssertConfiguration(expected.CoreConfiguration, actual.CoreConfiguration);
AssertConfiguration(expected.OozieConfiguration.ConfigurationCollection, actual.OozieConfiguration.ConfigurationCollection);
AssertConfiguration(expected.HiveConfiguration, actual.HiveConfiguration);
AssertConfiguration(expected.MapReduceConfiguration.ConfigurationCollection, actual.MapReduceConfiguration.ConfigurationCollection);
AssertConfiguration(expected.HdfsConfiguration, actual.HdfsConfiguration);
AssertConfiguration(expected.YarnConfiguration, actual.YarnConfiguration);
return true;
}
private static void ConfigHBaseComponent(HBaseComponent hbase, HDInsight.ClusterCreateParameters inputs)
{
hbase.HBaseConfXmlProperties.AddRange(
inputs.HBaseConfiguration.ConfigurationCollection.Select(prop => new Property { Name = prop.Key, Value = prop.Value }));
if (inputs.HBaseConfiguration.AdditionalLibraries != null)
{
hbase.AdditionalLibraries = new BlobContainerCredentialBackedResource()
{
AccountDnsName = inputs.HBaseConfiguration.AdditionalLibraries.Name,
BlobContainerName = inputs.HBaseConfiguration.AdditionalLibraries.Container,
Key = inputs.HBaseConfiguration.AdditionalLibraries.Key
};
}
}
/// <summary>
/// Generate ClusterCreateParameters object for 3.X cluster with Hadoop and HBase.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create3XClusterForMapReduceAndHBaseTemplate(HDInsight.ClusterCreateParameters inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var cluster = Create3XClusterFromMapReduceTemplate(inputs);
var hbaseMasterRole = cluster.Components.OfType<ZookeeperComponent>().Single().ZookeeperRole;
hbaseMasterRole.VMSize = VmSize.Medium;
//Add HBase component
HBaseComponent hbase = new HBaseComponent
{
MasterServerRole = hbaseMasterRole,
RegionServerRole = cluster.Components.OfType<HdfsComponent>().Single().WorkerNodeRole
};
ConfigHBaseComponent(hbase, inputs);
cluster.Components.Add(hbase);
return cluster;
}
private static void ConfigVirtualNetwork(ClusterCreateParameters cluster, HDInsight.ClusterCreateParameters inputs)
{
// Check if the virtual network configuration is partially set
if (string.IsNullOrEmpty(inputs.VirtualNetworkId) ^ string.IsNullOrEmpty(inputs.SubnetName))
{
if (inputs.VirtualNetworkId == null)
{
throw new ArgumentException("Subnet name is set however virtual network GUID is not set.");
}
else
{
throw new ArgumentException("Virtual newtork GUID is set however subnet name is not set.");
}
}
// Set virtual network configuration if is provided in the input
if (!string.IsNullOrEmpty(inputs.VirtualNetworkId) && !string.IsNullOrEmpty(inputs.SubnetName))
{
VirtualNetworkConfiguration virtualNetworkConf = new VirtualNetworkConfiguration();
virtualNetworkConf.VirtualNetworkSite = inputs.VirtualNetworkId;
foreach (var role in cluster.ClusterRoleCollection)
{
AddressAssignment aa = new AddressAssignment();
Subnet subnet = new Subnet();
subnet.Name = inputs.SubnetName;
aa.Subnets.Add(subnet);
aa.Role = role;
virtualNetworkConf.AddressAssignments.Add(aa);
}
cluster.VirtualNetworkConfiguration = virtualNetworkConf;
}
}
/// <summary>
/// Generate ClusterCreateParameters object for 3.X cluster with Hadoop and Storm.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create3XClusterForMapReduceAndStormTemplate(HDInsight.ClusterCreateParameters inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var cluster = Create3XClusterFromMapReduceTemplate(inputs);
var masterRole = cluster.Components.OfType<YarnComponent>().Single().ResourceManagerRole;
var workerRole = cluster.Components.OfType<YarnComponent>().Single().NodeManagerRole;
//Add Storm component
StormComponent storm = new StormComponent
{
MasterRole = masterRole,
WorkerRole = workerRole
};
ConfigStormComponent(storm, inputs);
cluster.Components.Add(storm);
return cluster;
}
public HDInsightJobSubmissionRestClient(IHDInsightSubscriptionCredentials credentials, HDInsight.IAbstractionContext context, bool ignoreSslErrors)
{
this.context = context;
this.credentials = credentials;
this.ignoreSslErrors = ignoreSslErrors;
}
private static void AddConfigActionComponent(CustomActionComponent configAction, HDInsight.ClusterCreateParameters inputs, ClusterRole headnodeRole, ClusterRole workernodeRole)
{
configAction.CustomActions = new CustomActionList();
// Converts config action from PS/SDK to wire contract.
foreach (ConfigAction ca in inputs.ConfigActions)
{
CustomAction newConfigAction;
// Based on the config action type defined in SDK, convert them to config action defined in wire contract.
ScriptAction sca = ca as ScriptAction;
if (sca != null)
{
newConfigAction = new ScriptCustomAction
{
Name = ca.Name,
Uri = sca.Uri,
Parameters = sca.Parameters
};
}
else
{
throw new NotSupportedException("No such config action supported.");
}
newConfigAction.ClusterRoleCollection = new ClusterRoleCollection();
// Add in cluster role collection for each config action.
foreach (ClusterNodeType clusterRoleType in ca.ClusterRoleCollection)
{
if (clusterRoleType == ClusterNodeType.HeadNode)
{
newConfigAction.ClusterRoleCollection.Add(headnodeRole);
}
else if (clusterRoleType == ClusterNodeType.DataNode)
{
newConfigAction.ClusterRoleCollection.Add(workernodeRole);
}
else
{
throw new NotSupportedException("No such node type supported.");
}
}
configAction.CustomActions.Add(newConfigAction);
}
}
public HDInsightManagementRestSimulatorClient(IHDInsightSubscriptionCredentials credentials, HDInsight.IAbstractionContext context)
{
this.context = context;
var cert = new X509Certificate2(IntegrationTestBase.TestCredentials.Certificate);
this.certificates.Add(cert.Thumbprint, cert);
this.subscriptions.Add(IntegrationTestBase.TestCredentials.SubscriptionId);
this.credentials = credentials;
this.Logger = new Logger();
}
private static void ConfigOozieComponent(OozieComponent oozie, HDInsight.ClusterCreateParameters inputs)
{
oozie.Configuration.AddRange(
inputs.OozieConfiguration.ConfigurationCollection.Select(prop => new Property { Name = prop.Key, Value = prop.Value }));
if (inputs.OozieConfiguration.AdditionalSharedLibraries != null)
{
oozie.AdditionalSharedLibraries = new BlobContainerCredentialBackedResource()
{
AccountDnsName = inputs.OozieConfiguration.AdditionalSharedLibraries.Name,
BlobContainerName = inputs.OozieConfiguration.AdditionalSharedLibraries.Container,
Key = inputs.OozieConfiguration.AdditionalSharedLibraries.Key
};
}
if (inputs.OozieConfiguration.AdditionalActionExecutorLibraries != null)
{
oozie.AdditionalActionExecutorLibraries = new BlobContainerCredentialBackedResource()
{
AccountDnsName = inputs.OozieConfiguration.AdditionalActionExecutorLibraries.Name,
BlobContainerName = inputs.OozieConfiguration.AdditionalActionExecutorLibraries.Container,
Key = inputs.OozieConfiguration.AdditionalActionExecutorLibraries.Key
};
}
if (inputs.OozieMetastore != null)
{
oozie.Metastore = new SqlAzureDatabaseCredentialBackedResource()
{
SqlServerName = inputs.OozieMetastore.Server,
Credentials =
new UsernamePasswordCredential() { Username = inputs.OozieMetastore.User, Password = inputs.OozieMetastore.Password },
DatabaseName = inputs.OozieMetastore.Database
};
}
}
private static IEnumerable<WabStorageAccountConfiguration> GetStorageAccounts(HDInsight.ClusterCreateParametersV2 cluster)
{
var storageAccounts = new List<WabStorageAccountConfiguration>();
storageAccounts.Add(
new WabStorageAccountConfiguration(cluster.DefaultStorageAccountName, cluster.DefaultStorageAccountKey, cluster.DefaultStorageContainer));
storageAccounts.AddRange(cluster.AdditionalStorageAccounts);
return storageAccounts;
}
private static Cluster CreateClusterFromCreateParameters(HDInsight.ClusterCreateParametersV2 clusterCreateParameters)
{
var clusterCreateParams = HDInsightClusterRequestGenerator.Create3XClusterFromMapReduceTemplate(clusterCreateParameters);
var cluster = new Cluster
{
ClusterRoleCollection = clusterCreateParams.ClusterRoleCollection,
CreatedTime = DateTime.UtcNow,
Error = null,
FullyQualifiedDnsName = clusterCreateParams.DnsName,
State = ClusterState.Running,
UpdatedTime = DateTime.UtcNow,
DnsName = clusterCreateParams.DnsName,
Components = clusterCreateParams.Components,
ExtensionData = clusterCreateParams.ExtensionData,
Location = clusterCreateParams.Location,
Version = clusterCreateParams.Version,
VirtualNetworkConfiguration = clusterCreateParams.VirtualNetworkConfiguration
};
return cluster;
}
