public void ICanCreateAClusterUsingPowerShellAndConfig()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
var coreConfig = new Hashtable();
coreConfig.Add("hadoop.logfile.size", "10000");
var yarnConfig = new Hashtable();
yarnConfig.Add("yarn.fakevalue", "12345");
var hbaseConfig = new Hashtable();
hbaseConfig.Add("hbase.blob.size", "12345");
var hbaseServiceConfig = new AzureHDInsightHBaseConfiguration
{
Configuration = hbaseConfig,
AdditionalLibraries =
new AzureHDInsightDefaultStorageAccount
{
StorageAccountKey = Guid.NewGuid().ToString(),
StorageAccountName = Guid.NewGuid().ToString(),
StorageContainerName = Guid.NewGuid().ToString()
}
};
string dnsName = this.GetRandomClusterName();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.EndProcessing();
int expected = getCommand.Output.Count();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig)
.WithParameter(CmdletConstants.ClusterSizeInNodes, 3)
.WithParameter(CmdletConstants.HeadNodeVMSize, NodeVMSize.Large)
.WithParameter(CmdletConstants.ClusterType, ClusterType.HBase)
.WithParameter(CmdletConstants.VirtualNetworkId, Guid.NewGuid().ToString())
.WithParameter(CmdletConstants.SubnetName, "fakeSubnet")
.AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage)
.WithParameter(CmdletConstants.StorageAccountName, TestCredentials.Environments[0].DefaultStorageAccount.Name)
.WithParameter(CmdletConstants.StorageAccountKey, TestCredentials.Environments[0].DefaultStorageAccount.Key)
.WithParameter(CmdletConstants.StorageContainerName, TestCredentials.Environments[0].DefaultStorageAccount.Container)
.AddCommand(CmdletConstants.AddAzureHDInsightConfigValues)
.WithParameter(CmdletConstants.CoreConfig, coreConfig)
.WithParameter(CmdletConstants.YarnConfig, yarnConfig)
.WithParameter(CmdletConstants.HBaseConfig, hbaseServiceConfig)
.AddCommand(CmdletConstants.NewAzureHDInsightCluster)
// Ensure that the subscription Id can be accepted as a guid as well as a string.
.WithParameter(CmdletConstants.Name, dnsName)
.WithParameter(CmdletConstants.Version, TestCredentials.WellKnownCluster.Version)
.WithParameter(CmdletConstants.Location, CmdletConstants.EastUs)
.WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword()))
.Invoke();
Assert.AreEqual(1, results.Results.Count);
Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name);
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.Name = dnsName;
getCommand.EndProcessing();
Assert.AreEqual(1, getCommand.Output.Count);
Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name);
results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster)
// Ensure that subscription id can be accepted as a sting as well as a guid.
.WithParameter(CmdletConstants.Name, dnsName)
.Invoke();
Assert.AreEqual(0, results.Results.Count);
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.EndProcessing();
Assert.AreEqual(expected, getCommand.Output.Count);
}
}
public void ICanCreateAClusterUsingPowerShellAndConfig_WithDebug()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
var coreConfig = new Hashtable();
coreConfig.Add("hadoop.logfile.size", "10000");
string dnsName = this.GetRandomClusterName();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
var getCommandLogWriter = new PowershellLogWriter();
BufferingLogWriterFactory.Instance = getCommandLogWriter;
IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.Logger = getCommandLogWriter;
getCommand.EndProcessing();
int expected = getCommand.Output.Count();
string expectedLogMessage = "Getting hdinsight clusters for subscriptionid : " + creds.SubscriptionId.ToString();
Assert.IsTrue(getCommandLogWriter.Buffer.Any(message => message.Contains(expectedLogMessage)));
BufferingLogWriterFactory.Reset();
var newClusterCommandLogWriter = new PowershellLogWriter();
BufferingLogWriterFactory.Instance = newClusterCommandLogWriter;
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig)
.WithParameter(CmdletConstants.ClusterSizeInNodes, 3)
.AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage)
.WithParameter(CmdletConstants.StorageAccountName, TestCredentials.Environments[0].DefaultStorageAccount.Name)
.WithParameter(CmdletConstants.StorageAccountKey, TestCredentials.Environments[0].DefaultStorageAccount.Key)
.WithParameter(CmdletConstants.StorageContainerName, TestCredentials.Environments[0].DefaultStorageAccount.Container)
.AddCommand(CmdletConstants.AddAzureHDInsightConfigValues)
.WithParameter(CmdletConstants.CoreConfig, coreConfig)
.AddCommand(CmdletConstants.NewAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, dnsName)
.WithParameter(CmdletConstants.Debug, null)
.WithParameter(CmdletConstants.Version, TestCredentials.WellKnownCluster.Version)
.WithParameter(CmdletConstants.Location, CmdletConstants.EastUs)
.WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword()))
.Invoke();
Assert.AreEqual(1, results.Results.Count);
Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name);
expectedLogMessage = string.Format(
CultureInfo.InvariantCulture, "Creating cluster '{0}' in location {1}", dnsName, CmdletConstants.EastUs);
Assert.IsTrue(newClusterCommandLogWriter.Buffer.Any(message => message.Contains(expectedLogMessage)));
BufferingLogWriterFactory.Reset();
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.Name = dnsName;
getCommand.EndProcessing();
Assert.AreEqual(1, getCommand.Output.Count);
Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name);
var deleteClusterCommandLogWriter = new PowershellLogWriter();
BufferingLogWriterFactory.Instance = deleteClusterCommandLogWriter;
results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, dnsName)
.WithParameter(CmdletConstants.Debug, null)
.Invoke();
Assert.AreEqual(0, results.Results.Count);
expectedLogMessage = string.Format(
CultureInfo.InvariantCulture, "Deleting cluster '{0}' in location {1}", dnsName, CmdletConstants.EastUs);
Assert.IsTrue(deleteClusterCommandLogWriter.Buffer.Any(message => message.Contains(expectedLogMessage)));
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.EndProcessing();
Assert.AreEqual(expected, getCommand.Output.Count);
BufferingLogWriterFactory.Reset();
}
}
public void ICanAddMultipleStorageAccountsUsingPowerShell()
{
IHDInsightCertificateCredential creds = GetValidCredentials();
string dnsName = this.GetRandomClusterName();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
var additionalStorageAccount = new WabStorageAccountConfiguration(
TestCredentials.Environments[0].AdditionalStorageAccounts[0].Name,
TestCredentials.Environments[0].AdditionalStorageAccounts[0].Key);
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig)
.WithParameter(CmdletConstants.ClusterSizeInNodes, 3)
.AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage)
.WithParameter(CmdletConstants.StorageAccountName, TestCredentials.Environments[0].DefaultStorageAccount.Name)
.WithParameter(CmdletConstants.StorageAccountKey, TestCredentials.Environments[0].DefaultStorageAccount.Key)
.WithParameter(CmdletConstants.StorageContainerName, TestCredentials.Environments[0].DefaultStorageAccount.Container)
.AddCommand(CmdletConstants.AddAzureHDInsightStorage)
.WithParameter(CmdletConstants.StorageAccountName, additionalStorageAccount.Name)
.WithParameter(CmdletConstants.StorageAccountKey, additionalStorageAccount.Key)
.AddCommand(CmdletConstants.NewAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, dnsName)
.WithParameter(CmdletConstants.Location, CmdletConstants.EastUs)
.WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword()))
.Invoke();
Assert.AreEqual(1, results.Results.Count);
Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name);
IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.Name = dnsName;
getCommand.EndProcessing();
Assert.AreEqual(1, getCommand.Output.Count);
Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name);
List <AzureHDInsightStorageAccount> storageAccounts = getCommand.Output.Last().StorageAccounts.ToList();
Assert.AreEqual(storageAccounts.Count, 1);
AzureHDInsightStorageAccount additionalStorageAccountFromOutput =
storageAccounts.FirstOrDefault(acc => acc.StorageAccountName == additionalStorageAccount.Name);
Assert.IsNotNull(additionalStorageAccountFromOutput);
Assert.AreEqual(additionalStorageAccount.Key, additionalStorageAccountFromOutput.StorageAccountKey);
results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster)
// Ensure that subscription id can be accepted as a sting as well as a guid.
.WithParameter(CmdletConstants.Name, dnsName)
.Invoke();
Assert.AreEqual(0, results.Results.Count);
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.Name = dnsName;
getCommand.EndProcessing();
Assert.AreEqual(0, getCommand.Output.Count);
}
}
public void ICanCreateAClusterUsingPowerShellAndConfig_New_Set_Add_Hive_Oozie_And_CoreConfig()
{
AzureTestCredentials creds = GetCredentials(TestCredentialsNames.Default);
string dnsName = this.GetRandomClusterName();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.EndProcessing();
int expected = getCommand.Output.Count();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig)
.WithParameter(CmdletConstants.ClusterSizeInNodes, 3)
.AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage)
.WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].DefaultStorageAccount.Name)
.WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].DefaultStorageAccount.Key)
.WithParameter(CmdletConstants.StorageContainerName, creds.Environments[0].DefaultStorageAccount.Container)
.AddCommand(CmdletConstants.AddAzureHDInsightStorage)
.WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].AdditionalStorageAccounts[0].Name)
.WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].AdditionalStorageAccounts[0].Key)
.AddCommand(CmdletConstants.AddAzureHDInsightMetastore)
.WithParameter(CmdletConstants.SqlAzureServerName, creds.Environments[0].HiveStores[0].SqlServer)
.WithParameter(CmdletConstants.DatabaseName, creds.Environments[0].HiveStores[0].Database)
.WithParameter(CmdletConstants.Credential, GetPSCredential(creds.AzureUserName, creds.AzurePassword))
.WithParameter(CmdletConstants.MetastoreType, AzureHDInsightMetastoreType.HiveMetastore)
.AddCommand(CmdletConstants.AddAzureHDInsightMetastore)
.WithParameter(CmdletConstants.SqlAzureServerName, creds.Environments[0].OozieStores[0].SqlServer)
.WithParameter(CmdletConstants.DatabaseName, creds.Environments[0].OozieStores[0].Database)
.WithParameter(CmdletConstants.Credential, GetPSCredential(creds.AzureUserName, creds.AzurePassword))
.WithParameter(CmdletConstants.MetastoreType, AzureHDInsightMetastoreType.OozieMetastore)
.AddCommand(CmdletConstants.NewAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, dnsName)
.WithParameter(CmdletConstants.Location, CmdletConstants.EastUs)
.WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword()))
.Invoke();
ClusterCreateParameters request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest;
Assert.IsNotNull(request.HiveMetastore);
Assert.IsNotNull(request.OozieMetastore);
Assert.AreEqual(1, results.Results.Count);
Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name);
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.Name = dnsName;
getCommand.EndProcessing();
Assert.AreEqual(1, getCommand.Output.Count);
Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name);
results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, dnsName)
.Invoke();
Assert.AreEqual(0, results.Results.Count);
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.EndProcessing();
Assert.AreEqual(expected, getCommand.Output.Count);
}
}
public void CanCallTheAddConfigValuesCmdletTestsCmdlet_PreserveConfig()
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
var coreConfig = new Hashtable();
var yarnConfig = new Hashtable();
var clusterConfig = new AzureHDInsightConfig
{
HiveMetastore =
new AzureHDInsightMetastore
{
MetastoreType = AzureHDInsightMetastoreType.HiveMetastore,
Credential = GetPSCredential("hadoop", Guid.NewGuid().ToString()),
DatabaseName = Guid.NewGuid().ToString(),
SqlAzureServerName = Guid.NewGuid().ToString()
},
OozieMetastore =
new AzureHDInsightMetastore
{
MetastoreType = AzureHDInsightMetastoreType.OozieMetastore,
Credential = GetPSCredential("hadoop", Guid.NewGuid().ToString()),
DatabaseName = Guid.NewGuid().ToString(),
SqlAzureServerName = Guid.NewGuid().ToString()
}
};
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.AddAzureHDInsightConfigValues)
.WithParameter(CmdletConstants.ClusterConfig, clusterConfig)
.WithParameter(CmdletConstants.CoreConfig, coreConfig)
.WithParameter(CmdletConstants.YarnConfig, yarnConfig)
.Invoke();
AzureHDInsightConfig config = results.Results.ToEnumerable <AzureHDInsightConfig>().First();
Assert.AreEqual(config.CoreConfiguration.Count, coreConfig.Count);
Assert.AreEqual(config.YarnConfiguration.Count, yarnConfig.Count);
foreach (object entry in coreConfig.Keys)
{
KeyValuePair <string, string> configUnderTest =
config.CoreConfiguration.FirstOrDefault(c => string.Equals(c.Key, entry.ToString(), StringComparison.Ordinal));
Assert.IsNotNull(configUnderTest, "Unable to find core config option with name '{0}'", entry);
Assert.AreEqual(coreConfig[entry], configUnderTest.Value, "value doesn't match for core config option with name '{0}'", entry);
}
foreach (object entry in yarnConfig.Keys)
{
KeyValuePair <string, string> configUnderTest =
config.YarnConfiguration.FirstOrDefault(c => string.Equals(c.Key, entry.ToString(), StringComparison.Ordinal));
Assert.IsNotNull(configUnderTest, "Unable to find yarn config option with name '{0}'", entry);
Assert.AreEqual(yarnConfig[entry], configUnderTest.Value, "value doesn't match for yarn config option with name '{0}'", entry);
}
Assert.AreEqual(clusterConfig.HiveMetastore.DatabaseName, config.HiveMetastore.DatabaseName);
Assert.AreEqual(clusterConfig.HiveMetastore.SqlAzureServerName, config.HiveMetastore.SqlAzureServerName);
Assert.AreEqual(clusterConfig.HiveMetastore.Credential.UserName, config.HiveMetastore.Credential.UserName);
Assert.AreEqual(clusterConfig.HiveMetastore.Credential.GetCleartextPassword(), config.HiveMetastore.Credential.GetCleartextPassword());
Assert.AreEqual(clusterConfig.OozieMetastore.DatabaseName, config.OozieMetastore.DatabaseName);
Assert.AreEqual(clusterConfig.OozieMetastore.SqlAzureServerName, config.OozieMetastore.SqlAzureServerName);
Assert.AreEqual(clusterConfig.OozieMetastore.Credential.UserName, config.OozieMetastore.Credential.UserName);
Assert.AreEqual(
clusterConfig.OozieMetastore.Credential.GetCleartextPassword(), config.OozieMetastore.Credential.GetCleartextPassword());
}
}
public void ICanCreateAClusterUsingPowerShellAndConfig_New_Set_Add_ScriptAction()
{
AzureTestCredentials creds = GetCredentials(TestCredentialsNames.Default);
string dnsName = this.GetRandomClusterName();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.EndProcessing();
int expected = getCommand.Output.Count();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig)
.WithParameter(CmdletConstants.ClusterSizeInNodes, 3)
.AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage)
.WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].DefaultStorageAccount.Name)
.WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].DefaultStorageAccount.Key)
.WithParameter(CmdletConstants.StorageContainerName, creds.Environments[0].DefaultStorageAccount.Container)
.AddCommand(CmdletConstants.AddAzureHDInsightStorage)
.WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].AdditionalStorageAccounts[0].Name)
.WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].AdditionalStorageAccounts[0].Key)
.AddCommand(CmdletConstants.AddAzureHDInsightScriptAction)
.WithParameter(CmdletConstants.ConfigActionName, "test1")
.WithParameter(CmdletConstants.ConfigActionClusterRoleCollection, "HeadNode")
.WithParameter(CmdletConstants.ScriptActionUri, "http://test1.com")
.WithParameter(CmdletConstants.ScriptActionParameters, "test1parameters")
.AddCommand(CmdletConstants.AddAzureHDInsightScriptAction)
.WithParameter(CmdletConstants.ConfigActionName, "test2")
.WithParameter(CmdletConstants.ConfigActionClusterRoleCollection, "HeadNode")
.WithParameter(CmdletConstants.ScriptActionUri, "http://test2.com")
.WithParameter(CmdletConstants.ScriptActionParameters, "test2parameters")
.AddCommand(CmdletConstants.NewAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, dnsName)
.WithParameter(CmdletConstants.Location, CmdletConstants.EastUs)
.WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword()))
.Invoke();
ClusterCreateParametersV2 request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest;
Assert.IsTrue(request.ConfigActions != null && request.ConfigActions.Count == 2);
Assert.IsTrue(
request.ConfigActions.ElementAt(0).Name == "test1" &&
request.ConfigActions.ElementAt(1).Name == "test2");
Assert.IsTrue(
request.ConfigActions.ElementAt(0).ClusterRoleCollection.Count == 1 &&
request.ConfigActions.ElementAt(1).ClusterRoleCollection.Count == 1);
Assert.AreEqual(1, results.Results.Count);
Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name);
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.Name = dnsName;
getCommand.EndProcessing();
Assert.AreEqual(1, getCommand.Output.Count);
Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name);
results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster)
.WithParameter(CmdletConstants.Name, dnsName)
.Invoke();
Assert.AreEqual(0, results.Results.Count);
getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet();
getCommand.CurrentSubscription = GetCurrentSubscription();
getCommand.EndProcessing();
Assert.AreEqual(expected, getCommand.Output.Count);
}
}
