public void ICanCreateAClusterUsingPowerShellAndConfig() { IHDInsightCertificateCredential creds = GetValidCredentials(); var coreConfig = new Hashtable(); coreConfig.Add("hadoop.logfile.size", "10000"); var yarnConfig = new Hashtable(); yarnConfig.Add("yarn.fakevalue", "12345"); var hbaseConfig = new Hashtable(); hbaseConfig.Add("hbase.blob.size", "12345"); var hbaseServiceConfig = new AzureHDInsightHBaseConfiguration { Configuration = hbaseConfig, AdditionalLibraries = new AzureHDInsightDefaultStorageAccount { StorageAccountKey = Guid.NewGuid().ToString(), StorageAccountName = Guid.NewGuid().ToString(), StorageContainerName = Guid.NewGuid().ToString() } }; string dnsName = this.GetRandomClusterName(); using (IRunspace runspace = this.GetPowerShellRunspace()) { IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.EndProcessing(); int expected = getCommand.Output.Count(); IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig) .WithParameter(CmdletConstants.ClusterSizeInNodes, 3) .WithParameter(CmdletConstants.HeadNodeVMSize, NodeVMSize.Large) .WithParameter(CmdletConstants.ClusterType, ClusterType.HBase) .WithParameter(CmdletConstants.VirtualNetworkId, Guid.NewGuid().ToString()) .WithParameter(CmdletConstants.SubnetName, "fakeSubnet") .AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage) .WithParameter(CmdletConstants.StorageAccountName, TestCredentials.Environments[0].DefaultStorageAccount.Name) .WithParameter(CmdletConstants.StorageAccountKey, TestCredentials.Environments[0].DefaultStorageAccount.Key) .WithParameter(CmdletConstants.StorageContainerName, TestCredentials.Environments[0].DefaultStorageAccount.Container) .AddCommand(CmdletConstants.AddAzureHDInsightConfigValues) .WithParameter(CmdletConstants.CoreConfig, coreConfig) .WithParameter(CmdletConstants.YarnConfig, yarnConfig) .WithParameter(CmdletConstants.HBaseConfig, hbaseServiceConfig) .AddCommand(CmdletConstants.NewAzureHDInsightCluster) // Ensure that the subscription Id can be accepted as a guid as well as a string. .WithParameter(CmdletConstants.Name, dnsName) .WithParameter(CmdletConstants.Version, TestCredentials.WellKnownCluster.Version) .WithParameter(CmdletConstants.Location, CmdletConstants.EastUs) .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword())) .Invoke(); Assert.AreEqual(1, results.Results.Count); Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.Name = dnsName; getCommand.EndProcessing(); Assert.AreEqual(1, getCommand.Output.Count); Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name); results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster) // Ensure that subscription id can be accepted as a sting as well as a guid. .WithParameter(CmdletConstants.Name, dnsName) .Invoke(); Assert.AreEqual(0, results.Results.Count); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.EndProcessing(); Assert.AreEqual(expected, getCommand.Output.Count); } }
public void ICanCreateAClusterUsingPowerShellAndConfig_WithDebug() { IHDInsightCertificateCredential creds = GetValidCredentials(); var coreConfig = new Hashtable(); coreConfig.Add("hadoop.logfile.size", "10000"); string dnsName = this.GetRandomClusterName(); using (IRunspace runspace = this.GetPowerShellRunspace()) { var getCommandLogWriter = new PowershellLogWriter(); BufferingLogWriterFactory.Instance = getCommandLogWriter; IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.Logger = getCommandLogWriter; getCommand.EndProcessing(); int expected = getCommand.Output.Count(); string expectedLogMessage = "Getting hdinsight clusters for subscriptionid : " + creds.SubscriptionId.ToString(); Assert.IsTrue(getCommandLogWriter.Buffer.Any(message => message.Contains(expectedLogMessage))); BufferingLogWriterFactory.Reset(); var newClusterCommandLogWriter = new PowershellLogWriter(); BufferingLogWriterFactory.Instance = newClusterCommandLogWriter; IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig) .WithParameter(CmdletConstants.ClusterSizeInNodes, 3) .AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage) .WithParameter(CmdletConstants.StorageAccountName, TestCredentials.Environments[0].DefaultStorageAccount.Name) .WithParameter(CmdletConstants.StorageAccountKey, TestCredentials.Environments[0].DefaultStorageAccount.Key) .WithParameter(CmdletConstants.StorageContainerName, TestCredentials.Environments[0].DefaultStorageAccount.Container) .AddCommand(CmdletConstants.AddAzureHDInsightConfigValues) .WithParameter(CmdletConstants.CoreConfig, coreConfig) .AddCommand(CmdletConstants.NewAzureHDInsightCluster) .WithParameter(CmdletConstants.Name, dnsName) .WithParameter(CmdletConstants.Debug, null) .WithParameter(CmdletConstants.Version, TestCredentials.WellKnownCluster.Version) .WithParameter(CmdletConstants.Location, CmdletConstants.EastUs) .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword())) .Invoke(); Assert.AreEqual(1, results.Results.Count); Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name); expectedLogMessage = string.Format( CultureInfo.InvariantCulture, "Creating cluster '{0}' in location {1}", dnsName, CmdletConstants.EastUs); Assert.IsTrue(newClusterCommandLogWriter.Buffer.Any(message => message.Contains(expectedLogMessage))); BufferingLogWriterFactory.Reset(); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.Name = dnsName; getCommand.EndProcessing(); Assert.AreEqual(1, getCommand.Output.Count); Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name); var deleteClusterCommandLogWriter = new PowershellLogWriter(); BufferingLogWriterFactory.Instance = deleteClusterCommandLogWriter; results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster) .WithParameter(CmdletConstants.Name, dnsName) .WithParameter(CmdletConstants.Debug, null) .Invoke(); Assert.AreEqual(0, results.Results.Count); expectedLogMessage = string.Format( CultureInfo.InvariantCulture, "Deleting cluster '{0}' in location {1}", dnsName, CmdletConstants.EastUs); Assert.IsTrue(deleteClusterCommandLogWriter.Buffer.Any(message => message.Contains(expectedLogMessage))); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.EndProcessing(); Assert.AreEqual(expected, getCommand.Output.Count); BufferingLogWriterFactory.Reset(); } }
public void ICanAddMultipleStorageAccountsUsingPowerShell() { IHDInsightCertificateCredential creds = GetValidCredentials(); string dnsName = this.GetRandomClusterName(); using (IRunspace runspace = this.GetPowerShellRunspace()) { var additionalStorageAccount = new WabStorageAccountConfiguration( TestCredentials.Environments[0].AdditionalStorageAccounts[0].Name, TestCredentials.Environments[0].AdditionalStorageAccounts[0].Key); IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig) .WithParameter(CmdletConstants.ClusterSizeInNodes, 3) .AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage) .WithParameter(CmdletConstants.StorageAccountName, TestCredentials.Environments[0].DefaultStorageAccount.Name) .WithParameter(CmdletConstants.StorageAccountKey, TestCredentials.Environments[0].DefaultStorageAccount.Key) .WithParameter(CmdletConstants.StorageContainerName, TestCredentials.Environments[0].DefaultStorageAccount.Container) .AddCommand(CmdletConstants.AddAzureHDInsightStorage) .WithParameter(CmdletConstants.StorageAccountName, additionalStorageAccount.Name) .WithParameter(CmdletConstants.StorageAccountKey, additionalStorageAccount.Key) .AddCommand(CmdletConstants.NewAzureHDInsightCluster) .WithParameter(CmdletConstants.Name, dnsName) .WithParameter(CmdletConstants.Location, CmdletConstants.EastUs) .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword())) .Invoke(); Assert.AreEqual(1, results.Results.Count); Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name); IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.Name = dnsName; getCommand.EndProcessing(); Assert.AreEqual(1, getCommand.Output.Count); Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name); List <AzureHDInsightStorageAccount> storageAccounts = getCommand.Output.Last().StorageAccounts.ToList(); Assert.AreEqual(storageAccounts.Count, 1); AzureHDInsightStorageAccount additionalStorageAccountFromOutput = storageAccounts.FirstOrDefault(acc => acc.StorageAccountName == additionalStorageAccount.Name); Assert.IsNotNull(additionalStorageAccountFromOutput); Assert.AreEqual(additionalStorageAccount.Key, additionalStorageAccountFromOutput.StorageAccountKey); results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster) // Ensure that subscription id can be accepted as a sting as well as a guid. .WithParameter(CmdletConstants.Name, dnsName) .Invoke(); Assert.AreEqual(0, results.Results.Count); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.Name = dnsName; getCommand.EndProcessing(); Assert.AreEqual(0, getCommand.Output.Count); } }
public void ICanCreateAClusterUsingPowerShellAndConfig_New_Set_Add_Hive_Oozie_And_CoreConfig() { AzureTestCredentials creds = GetCredentials(TestCredentialsNames.Default); string dnsName = this.GetRandomClusterName(); using (IRunspace runspace = this.GetPowerShellRunspace()) { IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.EndProcessing(); int expected = getCommand.Output.Count(); IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig) .WithParameter(CmdletConstants.ClusterSizeInNodes, 3) .AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage) .WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].DefaultStorageAccount.Name) .WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].DefaultStorageAccount.Key) .WithParameter(CmdletConstants.StorageContainerName, creds.Environments[0].DefaultStorageAccount.Container) .AddCommand(CmdletConstants.AddAzureHDInsightStorage) .WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].AdditionalStorageAccounts[0].Name) .WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].AdditionalStorageAccounts[0].Key) .AddCommand(CmdletConstants.AddAzureHDInsightMetastore) .WithParameter(CmdletConstants.SqlAzureServerName, creds.Environments[0].HiveStores[0].SqlServer) .WithParameter(CmdletConstants.DatabaseName, creds.Environments[0].HiveStores[0].Database) .WithParameter(CmdletConstants.Credential, GetPSCredential(creds.AzureUserName, creds.AzurePassword)) .WithParameter(CmdletConstants.MetastoreType, AzureHDInsightMetastoreType.HiveMetastore) .AddCommand(CmdletConstants.AddAzureHDInsightMetastore) .WithParameter(CmdletConstants.SqlAzureServerName, creds.Environments[0].OozieStores[0].SqlServer) .WithParameter(CmdletConstants.DatabaseName, creds.Environments[0].OozieStores[0].Database) .WithParameter(CmdletConstants.Credential, GetPSCredential(creds.AzureUserName, creds.AzurePassword)) .WithParameter(CmdletConstants.MetastoreType, AzureHDInsightMetastoreType.OozieMetastore) .AddCommand(CmdletConstants.NewAzureHDInsightCluster) .WithParameter(CmdletConstants.Name, dnsName) .WithParameter(CmdletConstants.Location, CmdletConstants.EastUs) .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword())) .Invoke(); ClusterCreateParameters request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest; Assert.IsNotNull(request.HiveMetastore); Assert.IsNotNull(request.OozieMetastore); Assert.AreEqual(1, results.Results.Count); Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.Name = dnsName; getCommand.EndProcessing(); Assert.AreEqual(1, getCommand.Output.Count); Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name); results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster) .WithParameter(CmdletConstants.Name, dnsName) .Invoke(); Assert.AreEqual(0, results.Results.Count); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.EndProcessing(); Assert.AreEqual(expected, getCommand.Output.Count); } }
public void CanCallTheAddConfigValuesCmdletTestsCmdlet_PreserveConfig() { using (IRunspace runspace = this.GetPowerShellRunspace()) { var coreConfig = new Hashtable(); var yarnConfig = new Hashtable(); var clusterConfig = new AzureHDInsightConfig { HiveMetastore = new AzureHDInsightMetastore { MetastoreType = AzureHDInsightMetastoreType.HiveMetastore, Credential = GetPSCredential("hadoop", Guid.NewGuid().ToString()), DatabaseName = Guid.NewGuid().ToString(), SqlAzureServerName = Guid.NewGuid().ToString() }, OozieMetastore = new AzureHDInsightMetastore { MetastoreType = AzureHDInsightMetastoreType.OozieMetastore, Credential = GetPSCredential("hadoop", Guid.NewGuid().ToString()), DatabaseName = Guid.NewGuid().ToString(), SqlAzureServerName = Guid.NewGuid().ToString() } }; IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.AddAzureHDInsightConfigValues) .WithParameter(CmdletConstants.ClusterConfig, clusterConfig) .WithParameter(CmdletConstants.CoreConfig, coreConfig) .WithParameter(CmdletConstants.YarnConfig, yarnConfig) .Invoke(); AzureHDInsightConfig config = results.Results.ToEnumerable <AzureHDInsightConfig>().First(); Assert.AreEqual(config.CoreConfiguration.Count, coreConfig.Count); Assert.AreEqual(config.YarnConfiguration.Count, yarnConfig.Count); foreach (object entry in coreConfig.Keys) { KeyValuePair <string, string> configUnderTest = config.CoreConfiguration.FirstOrDefault(c => string.Equals(c.Key, entry.ToString(), StringComparison.Ordinal)); Assert.IsNotNull(configUnderTest, "Unable to find core config option with name '{0}'", entry); Assert.AreEqual(coreConfig[entry], configUnderTest.Value, "value doesn't match for core config option with name '{0}'", entry); } foreach (object entry in yarnConfig.Keys) { KeyValuePair <string, string> configUnderTest = config.YarnConfiguration.FirstOrDefault(c => string.Equals(c.Key, entry.ToString(), StringComparison.Ordinal)); Assert.IsNotNull(configUnderTest, "Unable to find yarn config option with name '{0}'", entry); Assert.AreEqual(yarnConfig[entry], configUnderTest.Value, "value doesn't match for yarn config option with name '{0}'", entry); } Assert.AreEqual(clusterConfig.HiveMetastore.DatabaseName, config.HiveMetastore.DatabaseName); Assert.AreEqual(clusterConfig.HiveMetastore.SqlAzureServerName, config.HiveMetastore.SqlAzureServerName); Assert.AreEqual(clusterConfig.HiveMetastore.Credential.UserName, config.HiveMetastore.Credential.UserName); Assert.AreEqual(clusterConfig.HiveMetastore.Credential.GetCleartextPassword(), config.HiveMetastore.Credential.GetCleartextPassword()); Assert.AreEqual(clusterConfig.OozieMetastore.DatabaseName, config.OozieMetastore.DatabaseName); Assert.AreEqual(clusterConfig.OozieMetastore.SqlAzureServerName, config.OozieMetastore.SqlAzureServerName); Assert.AreEqual(clusterConfig.OozieMetastore.Credential.UserName, config.OozieMetastore.Credential.UserName); Assert.AreEqual( clusterConfig.OozieMetastore.Credential.GetCleartextPassword(), config.OozieMetastore.Credential.GetCleartextPassword()); } }
public void ICanCreateAClusterUsingPowerShellAndConfig_New_Set_Add_ScriptAction() { AzureTestCredentials creds = GetCredentials(TestCredentialsNames.Default); string dnsName = this.GetRandomClusterName(); using (IRunspace runspace = this.GetPowerShellRunspace()) { IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.EndProcessing(); int expected = getCommand.Output.Count(); IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig) .WithParameter(CmdletConstants.ClusterSizeInNodes, 3) .AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage) .WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].DefaultStorageAccount.Name) .WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].DefaultStorageAccount.Key) .WithParameter(CmdletConstants.StorageContainerName, creds.Environments[0].DefaultStorageAccount.Container) .AddCommand(CmdletConstants.AddAzureHDInsightStorage) .WithParameter(CmdletConstants.StorageAccountName, creds.Environments[0].AdditionalStorageAccounts[0].Name) .WithParameter(CmdletConstants.StorageAccountKey, creds.Environments[0].AdditionalStorageAccounts[0].Key) .AddCommand(CmdletConstants.AddAzureHDInsightScriptAction) .WithParameter(CmdletConstants.ConfigActionName, "test1") .WithParameter(CmdletConstants.ConfigActionClusterRoleCollection, "HeadNode") .WithParameter(CmdletConstants.ScriptActionUri, "http://test1.com") .WithParameter(CmdletConstants.ScriptActionParameters, "test1parameters") .AddCommand(CmdletConstants.AddAzureHDInsightScriptAction) .WithParameter(CmdletConstants.ConfigActionName, "test2") .WithParameter(CmdletConstants.ConfigActionClusterRoleCollection, "HeadNode") .WithParameter(CmdletConstants.ScriptActionUri, "http://test2.com") .WithParameter(CmdletConstants.ScriptActionParameters, "test2parameters") .AddCommand(CmdletConstants.NewAzureHDInsightCluster) .WithParameter(CmdletConstants.Name, dnsName) .WithParameter(CmdletConstants.Location, CmdletConstants.EastUs) .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword())) .Invoke(); ClusterCreateParametersV2 request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest; Assert.IsTrue(request.ConfigActions != null && request.ConfigActions.Count == 2); Assert.IsTrue( request.ConfigActions.ElementAt(0).Name == "test1" && request.ConfigActions.ElementAt(1).Name == "test2"); Assert.IsTrue( request.ConfigActions.ElementAt(0).ClusterRoleCollection.Count == 1 && request.ConfigActions.ElementAt(1).ClusterRoleCollection.Count == 1); Assert.AreEqual(1, results.Results.Count); Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.Name = dnsName; getCommand.EndProcessing(); Assert.AreEqual(1, getCommand.Output.Count); Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name); results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster) .WithParameter(CmdletConstants.Name, dnsName) .Invoke(); Assert.AreEqual(0, results.Results.Count); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.EndProcessing(); Assert.AreEqual(expected, getCommand.Output.Count); } }