public void CanAddHiveAdditionalLibrariesValues()
{
var config = new AzureHDInsightConfig();
IAddAzureHDInsightConfigValuesCommand addCoreConfigValues =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateAddConfig();
addCoreConfigValues.Hive.AdditionalLibraries = new AzureHDInsightDefaultStorageAccount
{
StorageAccountKey = Guid.NewGuid().ToString(),
StorageAccountName = Guid.NewGuid().ToString(),
StorageContainerName = Guid.NewGuid().ToString()
};
addCoreConfigValues.Config = config;
addCoreConfigValues.EndProcessing();
AzureHDInsightConfig newConfig = addCoreConfigValues.Output.First();
Assert.AreEqual(config.ClusterSizeInNodes, newConfig.ClusterSizeInNodes);
Assert.AreEqual(config.DefaultStorageAccount, newConfig.DefaultStorageAccount);
Assert.IsNotNull(newConfig.HiveConfiguration.AdditionalLibraries);
Assert.AreEqual(
newConfig.HiveConfiguration.AdditionalLibraries.Container, addCoreConfigValues.Hive.AdditionalLibraries.StorageContainerName);
Assert.AreEqual(newConfig.HiveConfiguration.AdditionalLibraries.Key, addCoreConfigValues.Hive.AdditionalLibraries.StorageAccountKey);
Assert.AreEqual(newConfig.HiveConfiguration.AdditionalLibraries.Name, addCoreConfigValues.Hive.AdditionalLibraries.StorageAccountName);
}
public void CanAddRoleCollectionToScriptAction()
{
var config = new AzureHDInsightConfig();
IAddAzureHDInsightScriptActionCommand scriptActionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateAddScriptAction();
scriptActionCommand.ClusterRoleCollection =
new ClusterNodeType[] { ClusterNodeType.HeadNode, ClusterNodeType.DataNode };
scriptActionCommand.Name = "test";
scriptActionCommand.Uri = new Uri("http://test.com");
scriptActionCommand.Parameters = "test parameters";
scriptActionCommand.Config = config;
scriptActionCommand.EndProcessing();
AzureHDInsightConfig newConfig = scriptActionCommand.Output.First();
Assert.AreEqual(config.ClusterSizeInNodes, newConfig.ClusterSizeInNodes);
Assert.AreEqual(config.DefaultStorageAccount, newConfig.DefaultStorageAccount);
Assert.IsTrue(config.ConfigActions.Count == newConfig.ConfigActions.Count && config.ConfigActions.Count == 1);
Assert.IsTrue(newConfig.ConfigActions.ElementAt(0) is AzureHDInsightScriptAction);
Assert.IsTrue(newConfig.ConfigActions.ElementAt(0).Name == "test" &&
((AzureHDInsightScriptAction)newConfig.ConfigActions.ElementAt(0)).Uri == new Uri("http://test.com") &&
((AzureHDInsightScriptAction)newConfig.ConfigActions.ElementAt(0)).Parameters == "test parameters");
Assert.IsTrue(Enumerable.SequenceEqual(newConfig.ConfigActions.ElementAt(0).ClusterRoleCollection,
new ClusterNodeType[] { ClusterNodeType.HeadNode, ClusterNodeType.DataNode }));
}
public void CanAddHdfsConfigValues()
{
var config = new AzureHDInsightConfig();
IAddAzureHDInsightConfigValuesCommand addCoreConfigValues =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateAddConfig();
addCoreConfigValues.Hdfs.Add("hadoop.log.file.size", "12345");
addCoreConfigValues.Config = config;
addCoreConfigValues.EndProcessing();
AzureHDInsightConfig newConfig = addCoreConfigValues.Output.First();
Assert.AreEqual(config.ClusterSizeInNodes, newConfig.ClusterSizeInNodes);
Assert.AreEqual(config.DefaultStorageAccount, newConfig.DefaultStorageAccount);
Assert.IsTrue(
newConfig.HdfsConfiguration.Any(configOption => configOption.Key == "hadoop.log.file.size" && configOption.Value == "12345"));
}
public void CanCallTheAddConfigValuesCmdletTestsCmdlet_PreserveConfig()
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
var coreConfig = new Hashtable();
var clusterConfig = new AzureHDInsightConfig
{
HiveMetastore =
new AzureHDInsightMetastore
{
MetastoreType = AzureHDInsightMetastoreType.HiveMetastore,
Credential = GetPSCredential("hadoop", Guid.NewGuid().ToString()),
DatabaseName = Guid.NewGuid().ToString(),
SqlAzureServerName = Guid.NewGuid().ToString()
},
OozieMetastore =
new AzureHDInsightMetastore
{
MetastoreType = AzureHDInsightMetastoreType.OozieMetastore,
Credential = GetPSCredential("hadoop", Guid.NewGuid().ToString()),
DatabaseName = Guid.NewGuid().ToString(),
SqlAzureServerName = Guid.NewGuid().ToString()
}
};
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.AddAzureHDInsightConfigValues)
.WithParameter(CmdletConstants.ClusterConfig, clusterConfig)
.WithParameter(CmdletConstants.CoreConfig, coreConfig)
.Invoke();
AzureHDInsightConfig config = results.Results.ToEnumerable<AzureHDInsightConfig>().First();
Assert.AreEqual(config.CoreConfiguration.Count, coreConfig.Count);
foreach (object entry in coreConfig.Keys)
{
KeyValuePair<string, string> configUnderTest =
config.CoreConfiguration.FirstOrDefault(c => string.Equals(c.Key, entry.ToString(), StringComparison.Ordinal));
Assert.IsNotNull(configUnderTest, "Unable to find core config option with name '{0}'", entry);
Assert.AreEqual(coreConfig[entry], configUnderTest.Value, "value doesn't match for core config option with name '{0}'", entry);
}
Assert.AreEqual(clusterConfig.HiveMetastore.DatabaseName, config.HiveMetastore.DatabaseName);
Assert.AreEqual(clusterConfig.HiveMetastore.SqlAzureServerName, config.HiveMetastore.SqlAzureServerName);
Assert.AreEqual(clusterConfig.HiveMetastore.Credential.UserName, config.HiveMetastore.Credential.UserName);
Assert.AreEqual(clusterConfig.HiveMetastore.Credential.GetCleartextPassword(), config.HiveMetastore.Credential.GetCleartextPassword());
Assert.AreEqual(clusterConfig.OozieMetastore.DatabaseName, config.OozieMetastore.DatabaseName);
Assert.AreEqual(clusterConfig.OozieMetastore.SqlAzureServerName, config.OozieMetastore.SqlAzureServerName);
Assert.AreEqual(clusterConfig.OozieMetastore.Credential.UserName, config.OozieMetastore.Credential.UserName);
Assert.AreEqual(
clusterConfig.OozieMetastore.Credential.GetCleartextPassword(), config.OozieMetastore.Credential.GetCleartextPassword());
}
}
public void CanAddOozieConfigValues()
{
var config = new AzureHDInsightConfig();
IAddAzureHDInsightConfigValuesCommand addCoreConfigValues =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateAddConfig();
addCoreConfigValues.Oozie.AdditionalSharedLibraries = new AzureHDInsightDefaultStorageAccount
{
StorageAccountKey = Guid.NewGuid().ToString(),
StorageAccountName = Guid.NewGuid().ToString(),
StorageContainerName = Guid.NewGuid().ToString()
};
addCoreConfigValues.Oozie.AdditionalActionExecutorLibraries = new AzureHDInsightDefaultStorageAccount
{
StorageAccountKey = Guid.NewGuid().ToString(),
StorageAccountName = Guid.NewGuid().ToString(),
StorageContainerName = Guid.NewGuid().ToString()
};
addCoreConfigValues.Oozie.Configuration.Add("hadoop.log.file.size", "12345");
addCoreConfigValues.Config = config;
addCoreConfigValues.EndProcessing();
AzureHDInsightConfig newConfig = addCoreConfigValues.Output.First();
Assert.AreEqual(config.ClusterSizeInNodes, newConfig.ClusterSizeInNodes);
Assert.AreEqual(config.DefaultStorageAccount, newConfig.DefaultStorageAccount);
Assert.IsTrue(
newConfig.OozieConfiguration.ConfigurationCollection.Any(
configOption => configOption.Key == "hadoop.log.file.size" && configOption.Value == "12345"));
Assert.IsNotNull(newConfig.OozieConfiguration.AdditionalSharedLibraries);
Assert.AreEqual(
newConfig.OozieConfiguration.AdditionalSharedLibraries.Container,
addCoreConfigValues.Oozie.AdditionalSharedLibraries.StorageContainerName);
Assert.AreEqual(
newConfig.OozieConfiguration.AdditionalSharedLibraries.Key, addCoreConfigValues.Oozie.AdditionalSharedLibraries.StorageAccountKey);
Assert.AreEqual(
newConfig.OozieConfiguration.AdditionalSharedLibraries.Name, addCoreConfigValues.Oozie.AdditionalSharedLibraries.StorageAccountName);
Assert.IsNotNull(newConfig.OozieConfiguration.AdditionalActionExecutorLibraries);
Assert.AreEqual(
newConfig.OozieConfiguration.AdditionalActionExecutorLibraries.Container,
addCoreConfigValues.Oozie.AdditionalActionExecutorLibraries.StorageContainerName);
Assert.AreEqual(
newConfig.OozieConfiguration.AdditionalActionExecutorLibraries.Key,
addCoreConfigValues.Oozie.AdditionalActionExecutorLibraries.StorageAccountKey);
Assert.AreEqual(
newConfig.OozieConfiguration.AdditionalActionExecutorLibraries.Name,
addCoreConfigValues.Oozie.AdditionalActionExecutorLibraries.StorageAccountName);
}
public void CanCallTheAddConfigValuesCmdletTestsCmdlet_HiveConfig_Multiple_Invokes()
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
var hiveConfig = new Hashtable();
hiveConfig.Add("hadoop.logfiles.size", "12345");
var hiveServiceConfig = new AzureHDInsightHiveConfiguration
{
Configuration = hiveConfig,
AdditionalLibraries =
new AzureHDInsightDefaultStorageAccount
{
StorageAccountKey = Guid.NewGuid().ToString(),
StorageAccountName = Guid.NewGuid().ToString(),
StorageContainerName = Guid.NewGuid().ToString()
}
};
var hiveConfig2 = new Hashtable();
hiveConfig.Add("hadoop.logfiles.size2", "12345");
var hiveServiceConfig2 = new AzureHDInsightHiveConfiguration
{
Configuration = hiveConfig,
AdditionalLibraries =
new AzureHDInsightDefaultStorageAccount
{
StorageAccountKey = Guid.NewGuid().ToString(),
StorageAccountName = Guid.NewGuid().ToString(),
StorageContainerName = Guid.NewGuid().ToString()
}
};
var results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.AddAzureHDInsightConfigValues)
.WithParameter(CmdletConstants.ClusterConfig, new AzureHDInsightConfig())
.WithParameter(CmdletConstants.HiveConfig, hiveServiceConfig)
.AddCommand(CmdletConstants.AddAzureHDInsightConfigValues)
.WithParameter(CmdletConstants.HiveConfig, hiveServiceConfig2)
.Invoke();
AzureHDInsightConfig config = results.Results.ToEnumerable <AzureHDInsightConfig>().First();
Assert.IsNotNull(config.HiveConfiguration.AdditionalLibraries);
Assert.AreEqual(config.HiveConfiguration.AdditionalLibraries.Container, hiveServiceConfig2.AdditionalLibraries.StorageContainerName);
Assert.AreEqual(config.HiveConfiguration.AdditionalLibraries.Key, hiveServiceConfig2.AdditionalLibraries.StorageAccountKey);
Assert.AreEqual(config.HiveConfiguration.AdditionalLibraries.Name, hiveServiceConfig2.AdditionalLibraries.StorageAccountName);
ValidateConfigurationOptions(hiveConfig, config.HiveConfiguration.ConfigurationCollection);
ValidateConfigurationOptions(hiveConfig2, config.HiveConfiguration.ConfigurationCollection);
}
}
public NewAzureHDInsightClusterConfigCommand()
{
_config = new AzureHDInsightConfig();
}
public void CanCallTheAddConfigValuesCmdletTestsCmdlet_PreserveConfig()
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
var coreConfig = new Hashtable();
var yarnConfig = new Hashtable();
var stormConfig = new Hashtable();
stormConfig.Add("storm.fakekey", "123");
var clusterConfig = new AzureHDInsightConfig
{
HiveMetastore =
new AzureHDInsightMetastore
{
MetastoreType = AzureHDInsightMetastoreType.HiveMetastore,
Credential = GetPSCredential("hadoop", Guid.NewGuid().ToString()),
DatabaseName = Guid.NewGuid().ToString(),
SqlAzureServerName = Guid.NewGuid().ToString()
},
OozieMetastore =
new AzureHDInsightMetastore
{
MetastoreType = AzureHDInsightMetastoreType.OozieMetastore,
Credential = GetPSCredential("hadoop", Guid.NewGuid().ToString()),
DatabaseName = Guid.NewGuid().ToString(),
SqlAzureServerName = Guid.NewGuid().ToString()
}
};
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.AddAzureHDInsightConfigValues)
.WithParameter(CmdletConstants.ClusterConfig, clusterConfig)
.WithParameter(CmdletConstants.CoreConfig, coreConfig)
.WithParameter(CmdletConstants.YarnConfig, yarnConfig)
.WithParameter(CmdletConstants.StormConfig, stormConfig)
.Invoke();
AzureHDInsightConfig config = results.Results.ToEnumerable <AzureHDInsightConfig>().First();
Assert.AreEqual(config.CoreConfiguration.Count, coreConfig.Count);
Assert.AreEqual(config.YarnConfiguration.Count, yarnConfig.Count);
Assert.AreEqual(config.StormConfiguration.Count, stormConfig.Count);
foreach (object entry in coreConfig.Keys)
{
KeyValuePair <string, string> configUnderTest =
config.CoreConfiguration.FirstOrDefault(c => string.Equals(c.Key, entry.ToString(), StringComparison.Ordinal));
Assert.IsNotNull(configUnderTest, "Unable to find core config option with name '{0}'", entry);
Assert.AreEqual(coreConfig[entry], configUnderTest.Value, "value doesn't match for core config option with name '{0}'", entry);
}
foreach (object entry in yarnConfig.Keys)
{
KeyValuePair <string, string> configUnderTest =
config.YarnConfiguration.FirstOrDefault(c => string.Equals(c.Key, entry.ToString(), StringComparison.Ordinal));
Assert.IsNotNull(configUnderTest, "Unable to find yarn config option with name '{0}'", entry);
Assert.AreEqual(yarnConfig[entry], configUnderTest.Value, "value doesn't match for yarn config option with name '{0}'", entry);
}
foreach (object entry in stormConfig.Keys)
{
KeyValuePair <string, string> configUnderTest =
config.StormConfiguration.FirstOrDefault(c => string.Equals(c.Key, entry.ToString(), StringComparison.Ordinal));
Assert.IsNotNull(configUnderTest, "Unable to find storm config option with name '{0}'", entry);
Assert.AreEqual(stormConfig[entry], configUnderTest.Value, "value doesn't match for storm config option with name '{0}'", entry);
}
Assert.AreEqual(clusterConfig.HiveMetastore.DatabaseName, config.HiveMetastore.DatabaseName);
Assert.AreEqual(clusterConfig.HiveMetastore.SqlAzureServerName, config.HiveMetastore.SqlAzureServerName);
Assert.AreEqual(clusterConfig.HiveMetastore.Credential.UserName, config.HiveMetastore.Credential.UserName);
Assert.AreEqual(clusterConfig.HiveMetastore.Credential.GetCleartextPassword(), config.HiveMetastore.Credential.GetCleartextPassword());
Assert.AreEqual(clusterConfig.OozieMetastore.DatabaseName, config.OozieMetastore.DatabaseName);
Assert.AreEqual(clusterConfig.OozieMetastore.SqlAzureServerName, config.OozieMetastore.SqlAzureServerName);
Assert.AreEqual(clusterConfig.OozieMetastore.Credential.UserName, config.OozieMetastore.Credential.UserName);
Assert.AreEqual(
clusterConfig.OozieMetastore.Credential.GetCleartextPassword(), config.OozieMetastore.Credential.GetCleartextPassword());
}
}
public void CanAddComplexScriptAction()
{
var config = new AzureHDInsightConfig();
IAddAzureHDInsightScriptActionCommand scriptActionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateAddScriptAction();
scriptActionCommand.ClusterRoleCollection =
new ClusterNodeType[] { ClusterNodeType.HeadNode };
scriptActionCommand.Config = config;
scriptActionCommand.EndProcessing();
AzureHDInsightConfig newConfig = scriptActionCommand.Output.First();
Assert.AreEqual(config.ClusterSizeInNodes, newConfig.ClusterSizeInNodes);
Assert.AreEqual(config.DefaultStorageAccount, newConfig.DefaultStorageAccount);
Assert.IsTrue(config.ConfigActions.Count == newConfig.ConfigActions.Count && config.ConfigActions.Count == 1);
Assert.IsTrue(newConfig.ConfigActions.ElementAt(0) is AzureHDInsightScriptAction);
Assert.IsTrue(Enumerable.SequenceEqual(newConfig.ConfigActions.ElementAt(0).ClusterRoleCollection,
new ClusterNodeType[] { ClusterNodeType.HeadNode }));
}