public void TestDatalakeStorageInfo()
{
string existingDatalakeStorageShortName = "test";
string existingDatalakeStorageNameFqdn = "test.azuredatalakestore.net";
string existingDatalakeStorageNameFqdnUri = "adl://" + existingDatalakeStorageNameFqdn;
// Test Datalake null/empty storageAccount name
TestAndAssert(() =>
{ return(new AzureDataLakeStoreInfo("", "/")); },
GetErrorMessage(Constants.ERROR_INPUT_CANNOT_BE_EMPTY, "storageAccountName")
);
// Test Datalake storageAccount shortname
var datalakeShortName = new AzureDataLakeStoreInfo(existingDatalakeStorageShortName, "/");
Assert.Equal(datalakeShortName.StorageAccountName, existingDatalakeStorageNameFqdn);
// Test Datalake storageAccount FullyQualifiedName
var datalakeFullName = new AzureDataLakeStoreInfo(existingDatalakeStorageNameFqdn, "/");
Assert.Equal(datalakeFullName.StorageAccountName, existingDatalakeStorageNameFqdn);
// Test Datalake storageAccount with url input
TestAndAssert(
() => { return(new AzureDataLakeStoreInfo(existingDatalakeStorageNameFqdnUri, "/")); },
GetErrorMessage(Constants.ERROR_SCHEME_SPECIFIED_IN_STORAGE_FQDN, "storageAccountName")
);
// Test Datalake null/empty storageRootPath
TestAndAssert(
() => { return(new AzureDataLakeStoreInfo(existingDatalakeStorageShortName, "")); },
GetErrorMessage(Constants.ERROR_INPUT_CANNOT_BE_EMPTY, "storageRootPath")
);
}
public static void ValidateStorageConfigurations(IReadOnlyDictionary <string, Dictionary <string, string> > configurations, StorageInfo defaultStorageInfo,
Dictionary <string, string> additionalStorageAccounts)
{
AzureStorageInfo azureStorage = defaultStorageInfo as AzureStorageInfo;
AzureDataLakeStoreInfo adlStorage = defaultStorageInfo as AzureDataLakeStoreInfo;
bool shouldValidateCoreConfig = azureStorage != null || adlStorage != null || additionalStorageAccounts.Any();
if (shouldValidateCoreConfig)
{
Assert.True(configurations.ContainsKey(ConfigurationKey.CoreSite));
Dictionary <string, string> coreConfig = configurations[ConfigurationKey.CoreSite];
Assert.True(coreConfig.ContainsKey(Constants.StorageConfigurations.DefaultFsKey));
if (azureStorage != null)
{
Assert.True(coreConfig.ContainsKey(string.Format(Constants.StorageConfigurations.WasbStorageAccountKeyFormat, azureStorage.StorageAccountName)));
}
else if (adlStorage != null)
{
Assert.True(coreConfig.ContainsKey(Constants.StorageConfigurations.AdlHostNameKey));
Assert.True(coreConfig.ContainsKey(Constants.StorageConfigurations.AdlMountPointKey));
}
if (additionalStorageAccounts != null && additionalStorageAccounts.Any())
{
foreach (KeyValuePair <string, string> additionalStorageAccount in additionalStorageAccounts)
{
Assert.True(coreConfig.ContainsKey(string.Format(Constants.StorageConfigurations.WasbStorageAccountKeyFormat, additionalStorageAccount.Key)));
}
}
}
}
private static void AddStorageAccountsToCoreConfig(string clusterName, ClusterCreateParameters createProperties, IDictionary <string, Dictionary <string, string> > configurations)
{
//Note: Only HDI >v2.1 is supported.
//Get existing core configs.
Dictionary <string, string> coreConfig = GetExistingConfigurationsForType(configurations, ConfigurationKey.CoreSite);
// Note: Only WASB and ADLS Gen 1 storage accounts will be populated directly into configurations.
// Other storage account types will be populated into StorageProfile.
AzureStorageInfo azureStorageAccountInfo = createProperties.DefaultStorageInfo as AzureStorageInfo;
AzureDataLakeStoreInfo azureDataLakeStorageInfo = createProperties.DefaultStorageInfo as AzureDataLakeStoreInfo;
if (azureStorageAccountInfo != null)
{
//Add configurations for default WASB storage.
string container = string.IsNullOrWhiteSpace(azureStorageAccountInfo.StorageContainer)
? clusterName
: azureStorageAccountInfo.StorageContainer;
coreConfig[Constants.StorageConfigurations.DefaultFsKey] = string.Format(Constants.StorageConfigurations.DefaultFsWasbValueFormat,
container, azureStorageAccountInfo.StorageAccountName);
string defaultStorageConfigKey = string.Format(Constants.StorageConfigurations.WasbStorageAccountKeyFormat, azureStorageAccountInfo.StorageAccountName);
if (!string.IsNullOrEmpty(azureStorageAccountInfo.StorageAccountKey))
{
coreConfig[defaultStorageConfigKey] = azureStorageAccountInfo.StorageAccountKey;
}
}
else if (azureDataLakeStorageInfo != null)
{
//Add configurations for default ADL storage.
coreConfig[Constants.StorageConfigurations.DefaultFsKey] = Constants.StorageConfigurations.DefaultFsAdlValue;
coreConfig[Constants.StorageConfigurations.AdlHostNameKey] = azureDataLakeStorageInfo.StorageAccountName;
coreConfig[Constants.StorageConfigurations.AdlMountPointKey] = azureDataLakeStorageInfo.StorageRootPath;
}
//Add additional storage accounts.
Dictionary <string, string> additionalStorageAccounts = createProperties.AdditionalStorageAccounts ?? new Dictionary <string, string>();
foreach (KeyValuePair <string, string> storageAccount in additionalStorageAccounts)
{
string configKey = string.Format(Constants.StorageConfigurations.WasbStorageAccountKeyFormat, storageAccount.Key);
coreConfig[configKey] = storageAccount.Value;
}
configurations[ConfigurationKey.CoreSite] = coreConfig;
}
