public static void ValidateStorageConfigurations(IReadOnlyDictionary <string, Dictionary <string, string> > configurations, StorageInfo defaultStorageInfo,
Dictionary <string, string> additionalStorageAccounts)
{
AzureStorageInfo azureStorage = defaultStorageInfo as AzureStorageInfo;
AzureDataLakeStoreInfo adlStorage = defaultStorageInfo as AzureDataLakeStoreInfo;
bool shouldValidateCoreConfig = azureStorage != null || adlStorage != null || additionalStorageAccounts.Any();
if (shouldValidateCoreConfig)
{
Assert.True(configurations.ContainsKey(ConfigurationKey.CoreSite));
Dictionary <string, string> coreConfig = configurations[ConfigurationKey.CoreSite];
Assert.True(coreConfig.ContainsKey(Constants.StorageConfigurations.DefaultFsKey));
if (azureStorage != null)
{
Assert.True(coreConfig.ContainsKey(string.Format(Constants.StorageConfigurations.WasbStorageAccountKeyFormat, azureStorage.StorageAccountName)));
}
else if (adlStorage != null)
{
Assert.True(coreConfig.ContainsKey(Constants.StorageConfigurations.AdlHostNameKey));
Assert.True(coreConfig.ContainsKey(Constants.StorageConfigurations.AdlMountPointKey));
}
if (additionalStorageAccounts != null && additionalStorageAccounts.Any())
{
foreach (KeyValuePair <string, string> additionalStorageAccount in additionalStorageAccounts)
{
Assert.True(coreConfig.ContainsKey(string.Format(Constants.StorageConfigurations.WasbStorageAccountKeyFormat, additionalStorageAccount.Key)));
}
}
}
}
private void SaveFileToAzure(string blobUrl, byte[] fileArray, AzureStorageInfo azureStorageInfo)
{
var blobStorage = new BlobStorage(azureStorageInfo.Account, azureStorageInfo.AccountKey,
azureStorageInfo.Container);
blobStorage.UploadFromByteArray(fileArray, blobUrl);
}
private static void AddDefaultStorageAccountToCoreConfig(string clusterName, ClusterCreateParameters clusterCreateParameters, Dictionary <string, string> coreConfig)
{
string coreConfigDefaultFSKey = "fs.defaultFS";
string coreConfigDefaultFSKeyFor_2_1_Clusters = "fs.default.name";
var defaultStorageAccountKey = (clusterCreateParameters.Version != null && clusterCreateParameters.Version.Equals("2.1"))
? coreConfigDefaultFSKeyFor_2_1_Clusters
: coreConfigDefaultFSKey;
var azureStorageAccountInfo = clusterCreateParameters.DefaultStorageInfo as AzureStorageInfo;
var azureDataLakeStorageInfo = clusterCreateParameters.DefaultStorageInfo as AzureDataLakeStoreInfo;
if (azureStorageAccountInfo != null)
{
if (string.IsNullOrWhiteSpace(azureStorageAccountInfo.StorageContainer))
{
var storageInfoWithContainerName = new AzureStorageInfo(azureStorageAccountInfo.StorageAccountName, azureStorageAccountInfo.StorageAccountKey, clusterName);
clusterCreateParameters.DefaultStorageInfo = storageInfoWithContainerName;
coreConfig[defaultStorageAccountKey] = storageInfoWithContainerName.StorageAccountUri;
}
else
{
coreConfig[defaultStorageAccountKey] = azureStorageAccountInfo.StorageAccountUri;
}
}
else if (azureDataLakeStorageInfo != null)
{
// setup the parameters required for DataLake containers
coreConfig[defaultStorageAccountKey] = "adl://home";
coreConfig["dfs.adls.home.hostname"] = azureDataLakeStorageInfo.StorageAccountName;
coreConfig["dfs.adls.home.mountpoint"] = azureDataLakeStorageInfo.StorageRootPath;
}
}
public void SetAzureStorageInfo(string account, string accountKey, string container)
{
StorageInfo = new AzureStorageInfo
{
Account = account,
AccountKey = accountKey,
Container = container
};
}
public void TestAzureStorageInfo()
{
string existingStorageName = "testblob";
string existingStorageNameFqdn = "testblob.blob.core.windows.net";
string mooncakeStorageNameFqdn = "testblob.blob.core.chinacloudapi.cn";
string existingStorageKey = "testtest==";
string containerName = "testcontainer";
string existingStorageNameFqdnUri = string.Format("wasb://{0}@{1}", containerName, existingStorageNameFqdn);
// Test for optional storageContainer
var testStorageInfoOptionalContainer = new AzureStorageInfo(existingStorageNameFqdn, existingStorageKey);
Assert.Equal(testStorageInfoOptionalContainer.StorageContainer, "");
var testStorageInfoContainer = new AzureStorageInfo(existingStorageNameFqdn, existingStorageKey, containerName);
Assert.Equal(testStorageInfoContainer.StorageContainer, containerName);
// Test storageAccountName is null
TestAndAssert(() =>
{ return(new AzureStorageInfo("", existingStorageKey, containerName)); },
GetErrorMessage(Constants.ERROR_INPUT_CANNOT_BE_EMPTY, "storageAccountName")
);
// Test storageAccountKey is null
TestAndAssert(() =>
{ return(new AzureStorageInfo(existingStorageNameFqdn, "", containerName)); },
GetErrorMessage(Constants.ERROR_INPUT_CANNOT_BE_EMPTY, "storageAccountKey")
);
// Test non-fully-qualified storageAccountName
var testStorageAccountShortName = new AzureStorageInfo(existingStorageName, existingStorageKey);
Assert.Equal(testStorageAccountShortName.StorageAccountName, existingStorageNameFqdn);
// Test fully-qualified storageAccountName
var testStorageAccountFullyQualifiedName = new AzureStorageInfo(existingStorageNameFqdn, existingStorageKey);
Assert.Equal(testStorageAccountFullyQualifiedName.StorageAccountName, existingStorageNameFqdn);
// Test fully-qualified-mooncake storageAccountName
var testStorageAccountMooncakeFullyQualifiedName = new AzureStorageInfo(mooncakeStorageNameFqdn, existingStorageKey);
Assert.Equal(testStorageAccountMooncakeFullyQualifiedName.StorageAccountName, mooncakeStorageNameFqdn);
// Test storageAccountName with url input
TestAndAssert(() =>
{ return(new AzureStorageInfo(existingStorageNameFqdnUri, existingStorageKey)); },
GetErrorMessage(Constants.ERROR_SCHEME_SPECIFIED_IN_STORAGE_FQDN, "storageAccountName")
);
// Test for StorageAccountUri
var testWasbStorageInfo = new AzureStorageInfo(existingStorageNameFqdn, existingStorageKey, containerName);
Assert.Equal(existingStorageNameFqdnUri, testWasbStorageInfo.StorageAccountUri);
}
private static void AddStorageAccountsToCoreConfig(string clusterName, ClusterCreateParameters createProperties, IDictionary <string, Dictionary <string, string> > configurations)
{
//Note: Only HDI >v2.1 is supported.
//Get existing core configs.
Dictionary <string, string> coreConfig = GetExistingConfigurationsForType(configurations, ConfigurationKey.CoreSite);
// Note: Only WASB and ADLS Gen 1 storage accounts will be populated directly into configurations.
// Other storage account types will be populated into StorageProfile.
AzureStorageInfo azureStorageAccountInfo = createProperties.DefaultStorageInfo as AzureStorageInfo;
AzureDataLakeStoreInfo azureDataLakeStorageInfo = createProperties.DefaultStorageInfo as AzureDataLakeStoreInfo;
if (azureStorageAccountInfo != null)
{
//Add configurations for default WASB storage.
string container = string.IsNullOrWhiteSpace(azureStorageAccountInfo.StorageContainer)
? clusterName
: azureStorageAccountInfo.StorageContainer;
coreConfig[Constants.StorageConfigurations.DefaultFsKey] = string.Format(Constants.StorageConfigurations.DefaultFsWasbValueFormat,
container, azureStorageAccountInfo.StorageAccountName);
string defaultStorageConfigKey = string.Format(Constants.StorageConfigurations.WasbStorageAccountKeyFormat, azureStorageAccountInfo.StorageAccountName);
if (!string.IsNullOrEmpty(azureStorageAccountInfo.StorageAccountKey))
{
coreConfig[defaultStorageConfigKey] = azureStorageAccountInfo.StorageAccountKey;
}
}
else if (azureDataLakeStorageInfo != null)
{
//Add configurations for default ADL storage.
coreConfig[Constants.StorageConfigurations.DefaultFsKey] = Constants.StorageConfigurations.DefaultFsAdlValue;
coreConfig[Constants.StorageConfigurations.AdlHostNameKey] = azureDataLakeStorageInfo.StorageAccountName;
coreConfig[Constants.StorageConfigurations.AdlMountPointKey] = azureDataLakeStorageInfo.StorageRootPath;
}
//Add additional storage accounts.
Dictionary <string, string> additionalStorageAccounts = createProperties.AdditionalStorageAccounts ?? new Dictionary <string, string>();
foreach (KeyValuePair <string, string> storageAccount in additionalStorageAccounts)
{
string configKey = string.Format(Constants.StorageConfigurations.WasbStorageAccountKeyFormat, storageAccount.Key);
coreConfig[configKey] = storageAccount.Value;
}
configurations[ConfigurationKey.CoreSite] = coreConfig;
}
private void UploadScriptToStorageAccount(ClusterCreateParameters createParams)
{
if (HDInsightManagementTestUtilities.IsRecordMode())
{
AzureStorageInfo storageInfo = createParams.DefaultStorageInfo as AzureStorageInfo;
string storageAccountName = storageInfo.StorageAccountName.Split('.')[0];
StorageCredentials creds = new StorageCredentials(storageAccountName, storageInfo.StorageAccountKey);
CloudStorageAccount storageAccount = new CloudStorageAccount(creds, true);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(FailingScriptLocationContainer);
container.CreateIfNotExistsAsync();
CloudBlockBlob blob = container.GetBlockBlobReference("failingscriptaction.sh");
using (FileStream fileStream = File.OpenRead(@"TestData/FailingScriptAction.sh"))
{
blob.UploadFromStreamAsync(fileStream).GetAwaiter().GetResult();
}
}
}
