private void AssertContainerMatchesNameProvided(ClusterCreateParametersExtended createParamsExtended, string name)
{
Dictionary<string,string> coresiteConfig;
if (JsonConvert.DeserializeObject<Dictionary<string, Dictionary<string, string>>>(createParamsExtended.Properties.ClusterDefinition.Configurations).TryGetValue("core-site", out coresiteConfig))
{
string value;
coresiteConfig.TryGetValue("fs.defaultFS", out value);
Assert.True(value.StartsWith("wasb://" + name + "@"), "Container does not match the name provided");
}
else
{
Assert.True(false, "Deserialization error");
}
}
private ClusterCreateParametersExtended GetExtendedClusterCreateParameters(
string clusterName, ClusterCreateParameters clusterCreateParameters)
{
var createParamsExtended = new ClusterCreateParametersExtended
{
Location = clusterCreateParameters.Location,
Properties = new ClusterCreateProperties
{
ClusterDefinition = new ClusterDefinition
{
ClusterType = clusterCreateParameters.ClusterType
},
ClusterVersion = clusterCreateParameters.Version,
OperatingSystemType = clusterCreateParameters.OSType
}
};
var configurations = GetConfigurations(clusterName, clusterCreateParameters);
if (clusterCreateParameters.HiveMetastore != null)
{
var metastoreConfig = GetMetastoreConfig(clusterCreateParameters.HiveMetastore, clusterCreateParameters.OSType, "Hive");
foreach (var configSet in metastoreConfig)
{
if (configurations.ContainsKey(configSet.Key))
{
foreach (var config in configSet.Value)
{
configurations[configSet.Key].Add(config.Key, config.Value);
}
}
else
{
configurations.Add(configSet.Key, configSet.Value);
}
}
}
if (clusterCreateParameters.OozieMetastore != null)
{
var metastoreConfig = GetMetastoreConfig(clusterCreateParameters.OozieMetastore, clusterCreateParameters.OSType, "oozie");
foreach (var configSet in metastoreConfig)
{
if (configurations.ContainsKey(configSet.Key))
{
foreach (var config in configSet.Value)
{
configurations[configSet.Key].Add(config.Key, config.Value);
}
}
else
{
configurations.Add(configSet.Key, configSet.Value);
}
}
}
var serializedConfig = JsonConvert.SerializeObject(configurations);
createParamsExtended.Properties.ClusterDefinition.Configurations = serializedConfig;
var roles = GetRoleCollection(clusterCreateParameters);
createParamsExtended.Properties.ComputeProfile = new ComputeProfile();
foreach (var role in roles)
{
role.SecurityProfile = clusterCreateParameters.SecurityProfile;
createParamsExtended.Properties.ComputeProfile.Roles.Add(role);
}
return createParamsExtended;
}
public static ClusterCreateParametersExtended GetIaasClusterSpec()
{
var cluster = new ClusterCreateParametersExtended
{
Location = "West US",
Properties = new ClusterCreateProperties
{
ClusterDefinition = new ClusterDefinition
{
ClusterType = "Hadoop"
},
ClusterVersion = "3.2",
OperatingSystemType = OSType.Linux
}
};
var coreConfigs = new Dictionary<string, string>
{
{"fs.defaultFS", string.Format("wasb://{0}@{1}", DefaultContainer, StorageAccountName)},
{
string.Format("fs.azure.account.key.{0}", StorageAccountName),
StorageAccountKey
}
};
var gatewayConfigs = new Dictionary<string, string>
{
{"restAuthCredential.isEnabled", "true"},
{"restAuthCredential.username", HttpUser},
{"restAuthCredential.password", HttpPassword}
};
var configurations = new Dictionary<string, Dictionary<string, string>>
{
{"core-site", coreConfigs},
{"gateway", gatewayConfigs}
};
var serializedConfig = JsonConvert.SerializeObject(configurations);
cluster.Properties.ClusterDefinition.Configurations = serializedConfig;
cluster.Tags.Add("tag1", "value1");
cluster.Tags.Add("tag2", "value2");
var sshPublicKeys = new List<SshPublicKey>();
var sshPublicKey = new SshPublicKey
{
CertificateData =
string.Format("ssh-rsa {0}", SshKey)
};
sshPublicKeys.Add(sshPublicKey);
var headNode = new Role
{
Name = "headnode",
TargetInstanceCount = 1,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
UserName = "******",
SshProfile = new SshProfile
{
SshPublicKeys = sshPublicKeys
}
}
},
VirtualNetworkProfile = new VirtualNetworkProfile
{
Id = "vnetid",
SubnetName = "subnetname"
}
};
var workerNode = new Role
{
Name = "workernode",
TargetInstanceCount = 1,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
UserName = "******",
SshProfile = new SshProfile
{
SshPublicKeys = sshPublicKeys
}
}
}
};
cluster.Properties.ComputeProfile = new ComputeProfile();
cluster.Properties.ComputeProfile.Roles.Add(headNode);
cluster.Properties.ComputeProfile.Roles.Add(workerNode);
return cluster;
}
public static ClusterCreateParametersExtended GetPaasClusterSpec()
{
var cluster = new ClusterCreateParametersExtended
{
Location = "West US",
Properties = new ClusterCreateProperties
{
ClusterDefinition = new ClusterDefinition
{
ClusterType = "Hadoop"
},
ClusterVersion = "3.1",
OperatingSystemType = OSType.Windows
}
};
var coreConfigs = new Dictionary<string, string>
{
{"fs.defaultFS", string.Format("wasb://{0}@{1}", DefaultContainer, StorageAccountName)},
{
string.Format("fs.azure.account.key.{0}", StorageAccountName), StorageAccountKey
}
};
var gatewayConfigs = new Dictionary<string, string>
{
{"restAuthCredential.isEnabled", "true"},
{"restAuthCredential.username", HttpUser},
{"restAuthCredential.password", HttpPassword}
};
var configurations = new Dictionary<string, Dictionary<string, string>>
{
{"core-site", coreConfigs},
{"gateway", gatewayConfigs}
};
var serializedConfig = JsonConvert.SerializeObject(configurations);
cluster.Properties.ClusterDefinition.Configurations = serializedConfig;
cluster.Tags.Add("tag1", "value1");
cluster.Tags.Add("tag2", "value2");
var headNode = new Role
{
Name = "headnode",
TargetInstanceCount = 2,
HardwareProfile = new HardwareProfile
{
VmSize = "ExtraLarge"
},
OsProfile = new OsProfile
{
WindowsOperatingSystemProfile = new WindowsOperatingSystemProfile
{
RdpSettings = new RdpSettings
{
UserName = RdpUser,
Password = RdpPassword,
ExpiryDate = new DateTime(2025, 3, 1)
}
}
}
};
var workerNode = new Role
{
Name = "workernode",
TargetInstanceCount = 5,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
}
};
cluster.Properties.ComputeProfile = new ComputeProfile();
cluster.Properties.ComputeProfile.Roles.Add(headNode);
cluster.Properties.ComputeProfile.Roles.Add(workerNode);
return cluster;
}
public static ClusterCreateParametersExtended AddConfigurations(ClusterCreateParametersExtended cluster, string configurationKey, Dictionary<string, string> configs)
{
string configurations = cluster.Properties.ClusterDefinition.Configurations;
var config = JsonConvert.DeserializeObject<Dictionary<string, Dictionary<String, string>>>(configurations);
config.Add(configurationKey, configs);
var serializedConfig = JsonConvert.SerializeObject(config);
cluster.Properties.ClusterDefinition.Configurations = serializedConfig;
return cluster;
}
/// <summary>
/// Creates a new HDInsight cluster with the specified parameters.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.IClusterOperations.
/// </param>
/// <param name='resourceGroupName'>
/// Required. The name of the resource group.
/// </param>
/// <param name='clusterName'>
/// Required. The name of the cluster.
/// </param>
/// <param name='clusterCreateParameters'>
/// Required. The cluster create request.
/// </param>
/// <returns>
/// The GetCluster operation response.
/// </returns>
public static Task<ClusterGetResponse> CreateAsync(this IClusterOperations operations, string resourceGroupName, string clusterName, ClusterCreateParametersExtended clusterCreateParameters)
{
return operations.CreateAsync(resourceGroupName, clusterName, clusterCreateParameters, CancellationToken.None);
}
/// <summary>
/// Creates a new HDInsight cluster with the specified parameters.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.IClusterOperations.
/// </param>
/// <param name='resourceGroupName'>
/// Required. The name of the resource group.
/// </param>
/// <param name='clusterName'>
/// Required. The name of the cluster.
/// </param>
/// <param name='clusterCreateParameters'>
/// Required. The cluster create request.
/// </param>
/// <returns>
/// The GetCluster operation response.
/// </returns>
public static ClusterGetResponse Create(this IClusterOperations operations, string resourceGroupName, string clusterName, ClusterCreateParametersExtended clusterCreateParameters)
{
return Task.Factory.StartNew((object s) =>
{
return ((IClusterOperations)s).CreateAsync(resourceGroupName, clusterName, clusterCreateParameters);
}
, operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult();
}
