public static void ValidateStorageConfigurations(IReadOnlyDictionary <string, Dictionary <string, string> > configurations, StorageInfo defaultStorageInfo, Dictionary <string, string> additionalStorageAccounts) { AzureStorageInfo azureStorage = defaultStorageInfo as AzureStorageInfo; AzureDataLakeStoreInfo adlStorage = defaultStorageInfo as AzureDataLakeStoreInfo; bool shouldValidateCoreConfig = azureStorage != null || adlStorage != null || additionalStorageAccounts.Any(); if (shouldValidateCoreConfig) { Assert.True(configurations.ContainsKey(ConfigurationKey.CoreSite)); Dictionary <string, string> coreConfig = configurations[ConfigurationKey.CoreSite]; Assert.True(coreConfig.ContainsKey(Constants.StorageConfigurations.DefaultFsKey)); if (azureStorage != null) { Assert.True(coreConfig.ContainsKey(string.Format(Constants.StorageConfigurations.WasbStorageAccountKeyFormat, azureStorage.StorageAccountName))); } else if (adlStorage != null) { Assert.True(coreConfig.ContainsKey(Constants.StorageConfigurations.AdlHostNameKey)); Assert.True(coreConfig.ContainsKey(Constants.StorageConfigurations.AdlMountPointKey)); } if (additionalStorageAccounts != null && additionalStorageAccounts.Any()) { foreach (KeyValuePair <string, string> additionalStorageAccount in additionalStorageAccounts) { Assert.True(coreConfig.ContainsKey(string.Format(Constants.StorageConfigurations.WasbStorageAccountKeyFormat, additionalStorageAccount.Key))); } } } }
private void SaveFileToAzure(string blobUrl, byte[] fileArray, AzureStorageInfo azureStorageInfo) { var blobStorage = new BlobStorage(azureStorageInfo.Account, azureStorageInfo.AccountKey, azureStorageInfo.Container); blobStorage.UploadFromByteArray(fileArray, blobUrl); }
private static void AddDefaultStorageAccountToCoreConfig(string clusterName, ClusterCreateParameters clusterCreateParameters, Dictionary <string, string> coreConfig) { string coreConfigDefaultFSKey = "fs.defaultFS"; string coreConfigDefaultFSKeyFor_2_1_Clusters = "fs.default.name"; var defaultStorageAccountKey = (clusterCreateParameters.Version != null && clusterCreateParameters.Version.Equals("2.1")) ? coreConfigDefaultFSKeyFor_2_1_Clusters : coreConfigDefaultFSKey; var azureStorageAccountInfo = clusterCreateParameters.DefaultStorageInfo as AzureStorageInfo; var azureDataLakeStorageInfo = clusterCreateParameters.DefaultStorageInfo as AzureDataLakeStoreInfo; if (azureStorageAccountInfo != null) { if (string.IsNullOrWhiteSpace(azureStorageAccountInfo.StorageContainer)) { var storageInfoWithContainerName = new AzureStorageInfo(azureStorageAccountInfo.StorageAccountName, azureStorageAccountInfo.StorageAccountKey, clusterName); clusterCreateParameters.DefaultStorageInfo = storageInfoWithContainerName; coreConfig[defaultStorageAccountKey] = storageInfoWithContainerName.StorageAccountUri; } else { coreConfig[defaultStorageAccountKey] = azureStorageAccountInfo.StorageAccountUri; } } else if (azureDataLakeStorageInfo != null) { // setup the parameters required for DataLake containers coreConfig[defaultStorageAccountKey] = "adl://home"; coreConfig["dfs.adls.home.hostname"] = azureDataLakeStorageInfo.StorageAccountName; coreConfig["dfs.adls.home.mountpoint"] = azureDataLakeStorageInfo.StorageRootPath; } }
public void SetAzureStorageInfo(string account, string accountKey, string container) { StorageInfo = new AzureStorageInfo { Account = account, AccountKey = accountKey, Container = container }; }
public void TestAzureStorageInfo() { string existingStorageName = "testblob"; string existingStorageNameFqdn = "testblob.blob.core.windows.net"; string mooncakeStorageNameFqdn = "testblob.blob.core.chinacloudapi.cn"; string existingStorageKey = "testtest=="; string containerName = "testcontainer"; string existingStorageNameFqdnUri = string.Format("wasb://{0}@{1}", containerName, existingStorageNameFqdn); // Test for optional storageContainer var testStorageInfoOptionalContainer = new AzureStorageInfo(existingStorageNameFqdn, existingStorageKey); Assert.Equal(testStorageInfoOptionalContainer.StorageContainer, ""); var testStorageInfoContainer = new AzureStorageInfo(existingStorageNameFqdn, existingStorageKey, containerName); Assert.Equal(testStorageInfoContainer.StorageContainer, containerName); // Test storageAccountName is null TestAndAssert(() => { return(new AzureStorageInfo("", existingStorageKey, containerName)); }, GetErrorMessage(Constants.ERROR_INPUT_CANNOT_BE_EMPTY, "storageAccountName") ); // Test storageAccountKey is null TestAndAssert(() => { return(new AzureStorageInfo(existingStorageNameFqdn, "", containerName)); }, GetErrorMessage(Constants.ERROR_INPUT_CANNOT_BE_EMPTY, "storageAccountKey") ); // Test non-fully-qualified storageAccountName var testStorageAccountShortName = new AzureStorageInfo(existingStorageName, existingStorageKey); Assert.Equal(testStorageAccountShortName.StorageAccountName, existingStorageNameFqdn); // Test fully-qualified storageAccountName var testStorageAccountFullyQualifiedName = new AzureStorageInfo(existingStorageNameFqdn, existingStorageKey); Assert.Equal(testStorageAccountFullyQualifiedName.StorageAccountName, existingStorageNameFqdn); // Test fully-qualified-mooncake storageAccountName var testStorageAccountMooncakeFullyQualifiedName = new AzureStorageInfo(mooncakeStorageNameFqdn, existingStorageKey); Assert.Equal(testStorageAccountMooncakeFullyQualifiedName.StorageAccountName, mooncakeStorageNameFqdn); // Test storageAccountName with url input TestAndAssert(() => { return(new AzureStorageInfo(existingStorageNameFqdnUri, existingStorageKey)); }, GetErrorMessage(Constants.ERROR_SCHEME_SPECIFIED_IN_STORAGE_FQDN, "storageAccountName") ); // Test for StorageAccountUri var testWasbStorageInfo = new AzureStorageInfo(existingStorageNameFqdn, existingStorageKey, containerName); Assert.Equal(existingStorageNameFqdnUri, testWasbStorageInfo.StorageAccountUri); }
private static void AddStorageAccountsToCoreConfig(string clusterName, ClusterCreateParameters createProperties, IDictionary <string, Dictionary <string, string> > configurations) { //Note: Only HDI >v2.1 is supported. //Get existing core configs. Dictionary <string, string> coreConfig = GetExistingConfigurationsForType(configurations, ConfigurationKey.CoreSite); // Note: Only WASB and ADLS Gen 1 storage accounts will be populated directly into configurations. // Other storage account types will be populated into StorageProfile. AzureStorageInfo azureStorageAccountInfo = createProperties.DefaultStorageInfo as AzureStorageInfo; AzureDataLakeStoreInfo azureDataLakeStorageInfo = createProperties.DefaultStorageInfo as AzureDataLakeStoreInfo; if (azureStorageAccountInfo != null) { //Add configurations for default WASB storage. string container = string.IsNullOrWhiteSpace(azureStorageAccountInfo.StorageContainer) ? clusterName : azureStorageAccountInfo.StorageContainer; coreConfig[Constants.StorageConfigurations.DefaultFsKey] = string.Format(Constants.StorageConfigurations.DefaultFsWasbValueFormat, container, azureStorageAccountInfo.StorageAccountName); string defaultStorageConfigKey = string.Format(Constants.StorageConfigurations.WasbStorageAccountKeyFormat, azureStorageAccountInfo.StorageAccountName); if (!string.IsNullOrEmpty(azureStorageAccountInfo.StorageAccountKey)) { coreConfig[defaultStorageConfigKey] = azureStorageAccountInfo.StorageAccountKey; } } else if (azureDataLakeStorageInfo != null) { //Add configurations for default ADL storage. coreConfig[Constants.StorageConfigurations.DefaultFsKey] = Constants.StorageConfigurations.DefaultFsAdlValue; coreConfig[Constants.StorageConfigurations.AdlHostNameKey] = azureDataLakeStorageInfo.StorageAccountName; coreConfig[Constants.StorageConfigurations.AdlMountPointKey] = azureDataLakeStorageInfo.StorageRootPath; } //Add additional storage accounts. Dictionary <string, string> additionalStorageAccounts = createProperties.AdditionalStorageAccounts ?? new Dictionary <string, string>(); foreach (KeyValuePair <string, string> storageAccount in additionalStorageAccounts) { string configKey = string.Format(Constants.StorageConfigurations.WasbStorageAccountKeyFormat, storageAccount.Key); coreConfig[configKey] = storageAccount.Value; } configurations[ConfigurationKey.CoreSite] = coreConfig; }
private void UploadScriptToStorageAccount(ClusterCreateParameters createParams) { if (HDInsightManagementTestUtilities.IsRecordMode()) { AzureStorageInfo storageInfo = createParams.DefaultStorageInfo as AzureStorageInfo; string storageAccountName = storageInfo.StorageAccountName.Split('.')[0]; StorageCredentials creds = new StorageCredentials(storageAccountName, storageInfo.StorageAccountKey); CloudStorageAccount storageAccount = new CloudStorageAccount(creds, true); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(FailingScriptLocationContainer); container.CreateIfNotExistsAsync(); CloudBlockBlob blob = container.GetBlockBlobReference("failingscriptaction.sh"); using (FileStream fileStream = File.OpenRead(@"TestData/FailingScriptAction.sh")) { blob.UploadFromStreamAsync(fileStream).GetAwaiter().GetResult(); } } }