public void ICanAddMultipleStorageAccountsUsingPowerShell() { IHDInsightCertificateCredential creds = GetValidCredentials(); string dnsName = this.GetRandomClusterName(); using (IRunspace runspace = this.GetPowerShellRunspace()) { var additionalStorageAccount = new WabStorageAccountConfiguration( TestCredentials.Environments[0].AdditionalStorageAccounts[0].Name, TestCredentials.Environments[0].AdditionalStorageAccounts[0].Key); IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig) .WithParameter(CmdletConstants.ClusterSizeInNodes, 3) .AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage) .WithParameter(CmdletConstants.StorageAccountName, TestCredentials.Environments[0].DefaultStorageAccount.Name) .WithParameter(CmdletConstants.StorageAccountKey, TestCredentials.Environments[0].DefaultStorageAccount.Key) .WithParameter(CmdletConstants.StorageContainerName, TestCredentials.Environments[0].DefaultStorageAccount.Container) .AddCommand(CmdletConstants.AddAzureHDInsightStorage) .WithParameter(CmdletConstants.StorageAccountName, additionalStorageAccount.Name) .WithParameter(CmdletConstants.StorageAccountKey, additionalStorageAccount.Key) .AddCommand(CmdletConstants.NewAzureHDInsightCluster) .WithParameter(CmdletConstants.Name, dnsName) .WithParameter(CmdletConstants.Location, CmdletConstants.EastUs) .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword())) .Invoke(); Assert.AreEqual(1, results.Results.Count); Assert.AreEqual(dnsName, results.Results.ToEnumerable<AzureHDInsightCluster>().First().Name); IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.Name = dnsName; getCommand.EndProcessing(); Assert.AreEqual(1, getCommand.Output.Count); Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name); List<AzureHDInsightStorageAccount> storageAccounts = getCommand.Output.Last().StorageAccounts.ToList(); Assert.AreEqual(storageAccounts.Count, 1); AzureHDInsightStorageAccount additionalStorageAccountFromOutput = storageAccounts.FirstOrDefault(acc => acc.StorageAccountName == additionalStorageAccount.Name); Assert.IsNotNull(additionalStorageAccountFromOutput); Assert.AreEqual(additionalStorageAccount.Key, additionalStorageAccountFromOutput.StorageAccountKey); results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster) // Ensure that subscription id can be accepted as a sting as well as a guid. .WithParameter(CmdletConstants.Name, dnsName) .Invoke(); Assert.AreEqual(0, results.Results.Count); getCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.Name = dnsName; getCommand.EndProcessing(); Assert.AreEqual(0, getCommand.Output.Count); } }
public async Task CannotAccessNonExistentPrivateUriInStorageAccountUsingUriEndpointValidator() { var storageCreds = new WabStorageAccountConfiguration( IntegrationTestBase.TestCredentials.Environments[0].DefaultStorageAccount.Name, IntegrationTestBase.TestCredentials.Environments[0].DefaultStorageAccount.Key); string fullPathToContainer = string.Format( "https://{0}/{1}/thisblobshouldnotexist", UriEndpointValidator.GetFullyQualifiedStorageAccountName( IntegrationTestBase.TestCredentials.Environments[0].DefaultStorageAccount.Name), IntegrationTestBase.TestCredentials.Environments[0].DefaultStorageAccount.Container); await UriEndpointValidator.ValidateAndResolveWasbScriptActionEndpointUri(new Uri(fullPathToContainer), storageCreds); }
public async Task CanAccessHttpUriInStorageAccountUsingUriEndpointValidator() { var storageCreds = new WabStorageAccountConfiguration( IntegrationTestBase.TestCredentials.Environments[0].DefaultStorageAccount.Name, IntegrationTestBase.TestCredentials.Environments[0].DefaultStorageAccount.Key); var fullPathToContainer = new Uri(string.Format( CultureInfo.InvariantCulture, "https://{0}/{1}/hive", UriEndpointValidator.GetFullyQualifiedStorageAccountName( IntegrationTestBase.TestCredentials.Environments[0].DefaultStorageAccount.Name), IntegrationTestBase.TestCredentials.Environments[0].DefaultStorageAccount.Container)); await UriEndpointValidator.ValidateAndResolveWasbScriptActionEndpointUri(fullPathToContainer, storageCreds); }
private static void ValidateGetCluster(AzureHDInsightCluster cluster) { Assert.AreEqual(TestCredentials.WellKnownCluster.DnsName, cluster.Name); Assert.AreEqual(TestCredentials.WellKnownCluster.Version, cluster.Version); WabStorageAccountConfiguration defaultStorageAccount = GetWellKnownStorageAccounts().First(); Assert.AreEqual(defaultStorageAccount.Key, cluster.DefaultStorageAccount.StorageAccountKey); Assert.AreEqual(defaultStorageAccount.Name, cluster.DefaultStorageAccount.StorageAccountName); Assert.AreEqual(defaultStorageAccount.Container, cluster.DefaultStorageAccount.StorageContainerName); foreach (WabStorageAccountConfiguration account in GetWellKnownStorageAccounts().Skip(1)) { AzureHDInsightStorageAccount deserializedAccount = cluster.StorageAccounts.FirstOrDefault(acc => acc.StorageAccountName == account.Name); Assert.IsNotNull(deserializedAccount, account.Name); Assert.AreEqual(account.Key, deserializedAccount.StorageAccountKey); } }
/// <summary> /// Validates, appends the FQDN suffix if required to storage accounts and creates the default cluster specified in <paramref name="details"/>. /// </summary> /// <param name="details">The details.</param> public static void ValidateAndResolveAsvAccountsAndPrep(ClusterCreateParametersV2 details) { var defaultStorageAccount = new WabStorageAccountConfiguration( details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer); // Flattens all the configurations into a single list for more uniform validation var asvList = ResolveStorageAccounts(details.AdditionalStorageAccounts).ToList(); asvList.Add(ResolveStorageAccount(defaultStorageAccount)); // Basic validation on the ASV configurations if (string.IsNullOrEmpty(details.DefaultStorageContainer)) { throw new InvalidOperationException("Invalid Container. Default Storage Account Container cannot be null or empty"); } if (asvList.Any(asv => string.IsNullOrEmpty(asv.Name) || string.IsNullOrEmpty(asv.Key))) { throw new InvalidOperationException("Invalid Azure Configuration. Credentials cannot be null or empty"); } if (asvList.GroupBy(asv => asv.Name).Count(group => group.Count() > 1) > 0) { throw new InvalidOperationException("Invalid Azure Storage credential. Duplicated values detected"); } // Validates that we can establish the connection to the ASV Names and the default container var client = ServiceLocator.Instance.Locate <IAsvValidatorClientFactory>().Create(); asvList.ForEach(asv => client.ValidateAccount(asv.Name, asv.Key).Wait()); var resolvedAccounts = ResolveStorageAccounts(details.AdditionalStorageAccounts); details.AdditionalStorageAccounts.Clear(); foreach (var resolvedAccount in resolvedAccounts) { details.AdditionalStorageAccounts.Add(resolvedAccount); } var resolvedDefaultStorageAccount = ResolveStorageAccount(defaultStorageAccount); details.DefaultStorageAccountName = resolvedDefaultStorageAccount.Name; client.CreateContainerIfNotExists(details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer).Wait(); }
public override void Initialize() { base.Initialize(); base.ApplyIndividualTestMockingOnly(); this.storageAccount = GetWellKnownStorageAccounts().First(); this.storageClient = new WabStorageAbstraction( new WindowsAzureStorageAccountCredentials { Key = this.storageAccount.Key, Name = this.storageAccount.Name, ContainerName = this.storageAccount.Container }); jobSubmissionClient = new HDInsightHadoopClient( new JobSubmissionCertificateCredential( Cluster.Value.SubscriptionId, GetValidCredentials().Certificate, Cluster.Value.Name)); this.CleanupCluster(); }
private static void ValidateGetCluster(AzureHDInsightCluster cluster) { Assert.AreEqual(TestCredentials.WellKnownCluster.DnsName, cluster.Name); Assert.AreEqual(TestCredentials.HadoopUserName, cluster.HttpUserName); Assert.AreEqual(TestCredentials.AzurePassword, cluster.HttpPassword); Assert.AreEqual(ClusterState.Running, cluster.State); Assert.IsFalse(string.IsNullOrEmpty(cluster.ConnectionUrl)); Assert.AreEqual(VersionStatus.Compatible.ToString(), cluster.VersionStatus); Assert.AreEqual(TestCredentials.WellKnownCluster.Version, cluster.Version); WabStorageAccountConfiguration defaultStorageAccount = GetWellKnownStorageAccounts().First(); Assert.AreEqual(defaultStorageAccount.Key, cluster.DefaultStorageAccount.StorageAccountKey); Assert.AreEqual(defaultStorageAccount.Name, cluster.DefaultStorageAccount.StorageAccountName); Assert.AreEqual(defaultStorageAccount.Container, cluster.DefaultStorageAccount.StorageContainerName); foreach (WabStorageAccountConfiguration account in GetWellKnownStorageAccounts().Skip(1)) { AzureHDInsightStorageAccount deserializedAccount = cluster.StorageAccounts.FirstOrDefault(acc => acc.StorageAccountName == account.Name); Assert.IsNotNull(deserializedAccount, account.Name); Assert.AreEqual(account.Key, deserializedAccount.StorageAccountKey); } }
internal static Uri GetStatusDirectoryPath(string statusDirectory, WabStorageAccountConfiguration wabStorageAccount, string userAccount, string fileName) { Uri statusDirectoryPath; if (statusDirectory.StartsWith(Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) || statusDirectory.StartsWith(Constants.WabsProtocol, StringComparison.OrdinalIgnoreCase) || statusDirectory.StartsWith(Constants.WabsProtocol, StringComparison.OrdinalIgnoreCase)) { statusDirectoryPath = new Uri(statusDirectory); } else if (statusDirectory.StartsWith("/", StringComparison.Ordinal)) { statusDirectoryPath = new Uri( string.Format( CultureInfo.InvariantCulture, "{0}{1}@{2}/{3}/{4}", Constants.WabsProtocolSchemeName, wabStorageAccount.Container, wabStorageAccount.Name, statusDirectory.TrimStart('/'), fileName)); } else { statusDirectoryPath = new Uri( string.Format( CultureInfo.InvariantCulture, "{0}{1}@{2}/user/{3}/{4}/{5}", Constants.WabsProtocolSchemeName, wabStorageAccount.Container, wabStorageAccount.Name, userAccount, statusDirectory.TrimStart('/'), fileName)); } return(statusDirectoryPath); }
/// <summary> /// Validates, appends the FQDN suffix if required to storage accounts and creates the default cluster specified in <paramref name="details"/>. /// </summary> /// <param name="details">The details.</param> public static void ValidateAndResolveAsvAccountsAndPrep(ClusterCreateParametersV2 details) { var defaultStorageAccount = new WabStorageAccountConfiguration( details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer); // Flattens all the configurations into a single list for more uniform validation var asvList = ResolveStorageAccounts(details.AdditionalStorageAccounts).ToList(); asvList.Add(ResolveStorageAccount(defaultStorageAccount)); // Basic validation on the ASV configurations if (string.IsNullOrEmpty(details.DefaultStorageContainer)) { throw new InvalidOperationException("Invalid Container. Default Storage Account Container cannot be null or empty"); } if (asvList.Any(asv => string.IsNullOrEmpty(asv.Name) || string.IsNullOrEmpty(asv.Key))) { throw new InvalidOperationException("Invalid Azure Configuration. Credentials cannot be null or empty"); } if (asvList.GroupBy(asv => asv.Name).Count(group => group.Count() > 1) > 0) { throw new InvalidOperationException("Invalid Azure Storage credential. Duplicated values detected"); } // Validates that we can establish the connection to the ASV Names and the default container var client = ServiceLocator.Instance.Locate<IAsvValidatorClientFactory>().Create(); asvList.ForEach(asv => client.ValidateAccount(asv.Name, asv.Key).Wait()); var resolvedAccounts = ResolveStorageAccounts(details.AdditionalStorageAccounts); details.AdditionalStorageAccounts.Clear(); foreach (var resolvedAccount in resolvedAccounts) { details.AdditionalStorageAccounts.Add(resolvedAccount); } var resolvedDefaultStorageAccount = ResolveStorageAccount(defaultStorageAccount); details.DefaultStorageAccountName = resolvedDefaultStorageAccount.Name; client.CreateContainerIfNotExists(details.DefaultStorageAccountName, details.DefaultStorageAccountKey, details.DefaultStorageContainer).Wait(); }
private AzureHDInsightHiveJobDefinition UploadFileToStorage( AzureHDInsightHiveJobDefinition jobDefinition, AzureHDInsightClusterConnection currentConnection) { currentConnection.Cluster.DefaultStorageAccount.ArgumentNotNull("DefaultStorageAccount"); WabStorageAccountConfiguration storageAccount = currentConnection.Cluster.DefaultStorageAccount.ToWabStorageAccountConfiguration(); var storageHandler = ServiceLocator.Instance.Locate <IAzureHDInsightStorageHandlerFactory>().Create(storageAccount); string hiveQueryFilePath = string.Format( CultureInfo.InvariantCulture, HiveQueryFileStoragePath, currentConnection.Cluster.DefaultStorageAccount.StorageAccountName, currentConnection.Cluster.DefaultStorageAccount.StorageContainerName, currentConnection.Cluster.HttpUserName, Guid.NewGuid().ToString("N")); using (Stream hiveQueryFileStream = this.GetStreamForText(jobDefinition.Query)) { var hiveQueryFileUri = new Uri(hiveQueryFilePath, UriKind.RelativeOrAbsolute); storageHandler.UploadFile(hiveQueryFileUri, hiveQueryFileStream); jobDefinition.Query = string.Empty; jobDefinition.File = storageHandler.GetStoragePath(hiveQueryFileUri).OriginalString; } return(jobDefinition); }
internal static WabStorageAccountConfiguration ResolveStorageAccount(WabStorageAccountConfiguration storageAccount) { return new WabStorageAccountConfiguration( GetFullyQualifiedStorageAccountName(storageAccount.Name), storageAccount.Key, storageAccount.Container); }
public static async Task ValidateAndResolveWasbScriptActionEndpointUri(Uri uri, WabStorageAccountConfiguration storage) { if (uri == null) { throw new ArgumentNullException("uri"); } if (storage == null) { throw new ArgumentNullException("storage"); } // Try to check the existence of the script in the URI specified. var storageAbstractionCreds = new WindowsAzureStorageAccountCredentials() { Key = storage.Key, Name = GetFullyQualifiedStorageAccountName(storage.Name), }; var storageAbstraction = new WabStorageAbstraction(storageAbstractionCreds); bool exists = false; try { // Firstly converts the URI to wasb style and then test it against wasb storage for existence. exists = await storageAbstraction.Exists(ConvertToWasbUriIfNeeded(uri)); } catch (Exception e) { throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "Script action URI: {0} cannot be retrieved correctly. Inner exception: {1}", uri.AbsoluteUri, e.Message), e); } if (!exists) { throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "Script action URI: {0} cannot be retrieved correctly because it does not exist", uri.AbsoluteUri)); } }
public void InternalValidation_PayloadConverter_SerializationListContainersResult_WithoutErrorWithExtendedError() { var storageAccount = new WabStorageAccountConfiguration(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); var container1 = new ClusterDetails(GetRandomClusterName(), "ClusterStorageProvisioned") { CreatedDate = DateTime.Now, ConnectionUrl = @"https://some/long/uri/", HttpUserName = "******", Location = "West US", ClusterSizeInNodes = 10, Error = new ClusterErrorStatus(10, "error", "create"), Version = IntegrationTestBase.TestCredentials.WellKnownCluster.Version, }; container1.DefaultStorageAccount = storageAccount; container1.AdditionalStorageAccounts = new List<WabStorageAccountConfiguration>() { new WabStorageAccountConfiguration(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString()), new WabStorageAccountConfiguration(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString()) }; var originalContainers = new Collection<ClusterDetails> { container1 }; Guid subscriptionId = new Guid(); var payload = ServerSerializer.SerializeListContainersResult(originalContainers, "namespace", false, true); var finalContainers = new PayloadConverter().DeserializeListContainersResult(payload, "namespace", subscriptionId); Assert.AreEqual(originalContainers.Count, finalContainers.Count); var deserializedCluster = finalContainers.FirstOrDefault(cluster => cluster.Name == container1.Name); Assert.IsNotNull(deserializedCluster); Assert.AreEqual(deserializedCluster.SubscriptionId, subscriptionId); Assert.AreEqual(deserializedCluster.Error.Message, "error"); Assert.AreEqual(deserializedCluster.Error.HttpCode, 10); Assert.AreEqual(deserializedCluster.Error.OperationType, "create"); }
public void InternalValidation_PayloadConverter_SerializationListContainersResult() { var storageAccount = new WabStorageAccountConfiguration(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); // Creates two random containers var container1 = new ClusterDetails(GetRandomClusterName(), "Running") { CreatedDate = DateTime.Now, ConnectionUrl = @"https://some/long/uri/", HttpUserName = "******", Location = "East US", ClusterSizeInNodes = 20, Version = IntegrationTestBase.TestCredentials.WellKnownCluster.Version }; container1.DefaultStorageAccount = storageAccount; container1.AdditionalStorageAccounts = new List<WabStorageAccountConfiguration>() { new WabStorageAccountConfiguration(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString()), new WabStorageAccountConfiguration(Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString()) }; var container2 = new ClusterDetails(GetRandomClusterName(), "ClusterStorageProvisioned") { CreatedDate = DateTime.Now, ConnectionUrl = @"https://some/long/uri/", HttpUserName = "******", Location = "West US", ClusterSizeInNodes = 10, Error = new ClusterErrorStatus(10, "error", "create"), Version = IntegrationTestBase.TestCredentials.WellKnownCluster.Version }; var originalContainers = new Collection<ClusterDetails> { container1, container2 }; // Roundtrip serialize\deserialize Guid subscriptionId = new Guid(); var payload = ServerSerializer.SerializeListContainersResult(originalContainers, "namespace", true, false); var finalContainers = new PayloadConverter().DeserializeListContainersResult(payload, "namespace", subscriptionId); // Compares the lists Assert.AreEqual(originalContainers.Count, finalContainers.Count); foreach (var expectedCluster in originalContainers) { var deserializedCluster = finalContainers.FirstOrDefault(cluster => cluster.Name == expectedCluster.Name); Assert.IsNotNull(deserializedCluster); Assert.AreEqual(deserializedCluster.SubscriptionId, subscriptionId); Assert.IsTrue(Equals(expectedCluster, deserializedCluster), "Failed to deserialize cluster pigJobCreateParameters {0}", expectedCluster.Name); } }
public void ICanAddMultipleStorageAccountsUsingPowerShell() { IHDInsightCertificateCredential creds = GetValidCredentials(); string dnsName = this.GetRandomClusterName(); using (IRunspace runspace = this.GetPowerShellRunspace()) { var additionalStorageAccount = new WabStorageAccountConfiguration( TestCredentials.Environments[0].AdditionalStorageAccounts[0].Name, TestCredentials.Environments[0].AdditionalStorageAccounts[0].Key); IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightClusterConfig) .WithParameter(CmdletConstants.ClusterSizeInNodes, 3) .AddCommand(CmdletConstants.SetAzureHDInsightDefaultStorage) .WithParameter(CmdletConstants.StorageAccountName, TestCredentials.Environments[0].DefaultStorageAccount.Name) .WithParameter(CmdletConstants.StorageAccountKey, TestCredentials.Environments[0].DefaultStorageAccount.Key) .WithParameter(CmdletConstants.StorageContainerName, TestCredentials.Environments[0].DefaultStorageAccount.Container) .AddCommand(CmdletConstants.AddAzureHDInsightStorage) .WithParameter(CmdletConstants.StorageAccountName, additionalStorageAccount.Name) .WithParameter(CmdletConstants.StorageAccountKey, additionalStorageAccount.Key) .AddCommand(CmdletConstants.NewAzureHDInsightCluster) .WithParameter(CmdletConstants.Name, dnsName) .WithParameter(CmdletConstants.Location, CmdletConstants.EastUs) .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword())) .Invoke(); Assert.AreEqual(1, results.Results.Count); Assert.AreEqual(dnsName, results.Results.ToEnumerable <AzureHDInsightCluster>().First().Name); IGetAzureHDInsightClusterCommand getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.Name = dnsName; getCommand.EndProcessing(); Assert.AreEqual(1, getCommand.Output.Count); Assert.AreEqual(dnsName, getCommand.Output.ElementAt(0).Name); List <AzureHDInsightStorageAccount> storageAccounts = getCommand.Output.Last().StorageAccounts.ToList(); Assert.AreEqual(storageAccounts.Count, 1); AzureHDInsightStorageAccount additionalStorageAccountFromOutput = storageAccounts.FirstOrDefault(acc => acc.StorageAccountName == additionalStorageAccount.Name); Assert.IsNotNull(additionalStorageAccountFromOutput); Assert.AreEqual(additionalStorageAccount.Key, additionalStorageAccountFromOutput.StorageAccountKey); results = runspace.NewPipeline().AddCommand(CmdletConstants.RemoveAzureHDInsightCluster) // Ensure that subscription id can be accepted as a sting as well as a guid. .WithParameter(CmdletConstants.Name, dnsName) .Invoke(); Assert.AreEqual(0, results.Results.Count); getCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGet(); getCommand.CurrentSubscription = GetCurrentSubscription(); getCommand.Name = dnsName; getCommand.EndProcessing(); Assert.AreEqual(0, getCommand.Output.Count); } }
public IAzureHDInsightStorageHandler Create(WabStorageAccountConfiguration storageAccountConfiguration) { return new AzureHDInsightStorageHandler(storageAccountConfiguration); }
public IAzureHDInsightStorageHandler Create(WabStorageAccountConfiguration storageAccountConfiguration) { return(new AzureHDInsightStorageHandler(storageAccountConfiguration)); }
public AzureHDInsightStorageHandler(WabStorageAccountConfiguration storageAccountConfiguration) { this.storageAccountConfiguration = storageAccountConfiguration; }
internal static WabStorageAccountConfiguration ResolveStorageAccount(WabStorageAccountConfiguration storageAccount) { return(new WabStorageAccountConfiguration( GetFullyQualifiedStorageAccountName(storageAccount.Name), storageAccount.Key, storageAccount.Container)); }