public static ClusterDetails CreateClusterIfNotExists(ClusterType clusterType)
{
var name = AppConfig.AzureResourceName + clusterType.ToString().ToLowerInvariant();
LOG.Info(String.Format("Creating Cluster: Name = {0}, Type = {1}", name, clusterType));
var parameters = new ClusterCreateParametersV2()
{
Name = name,
ClusterSizeInNodes = HDInsightClusterSize,
Location = AppConfig.AzureResourceLocation,
ClusterType = clusterType,
HeadNodeSize = "Large",
DataNodeSize = "Large",
ZookeeperNodeSize = "Medium",
UserName = AppConfig.AzureResourceUsername,
Password = AppConfig.AzureResourcePassword,
DefaultStorageAccountName = AzureStorageHelper.Name,
DefaultStorageAccountKey = AzureStorageHelper.PrimaryKey,
DefaultStorageContainer = AppConfig.AzureResourceName,
Version = "3.2",
//VirtualNetworkId = VirtualNetworkHelper.VNetId,
//SubnetName = VirtualNetworkHelper.SubnetName,
};
return(CreateClusterIfNotExists(parameters));
}
public static void DeleteClusterIfExists(ClusterType clusterType)
{
var name = AppConfig.AzureResourceName + clusterType.ToString().ToLowerInvariant();
LOG.Info(String.Format("Deleting Cluster: Name = {0}, Type = {1}", name, clusterType));
DeleteClusterIfExists(name);
}
public override Request Request(Request request)
{
request.AddDataItem((byte)OperationType.DATACLUSTER_ADD);
request.AddDataItem(request.SessionId);
if (OClient.ProtocolVersion < 24)
{
request.AddDataItem(ClusterType.ToString().ToUpper());
}
request.AddDataItem(ClusterName);
if (OClient.ProtocolVersion < 24)
{
if (OClient.ProtocolVersion >= 10 || ClusterType == OClusterType.Physical)
{
request.AddDataItem("");
}
if (OClient.ProtocolVersion >= 10)
{
request.AddDataItem("default");
}
else
{
request.AddDataItem((short)-1);
}
}
if (OClient.ProtocolVersion >= 18)
{
request.AddDataItem((short)-1); //clusterid
}
return(request);
}
public override Request Request(Request request)
{
base.Request(request);
if (OClient.ProtocolVersion < 24)
{
request.AddDataItem(ClusterType.ToString().ToUpper());
}
request.AddDataItem(ClusterName);
if (OClient.ProtocolVersion < 24)
{
if (OClient.ProtocolVersion >= 10 || ClusterType == OClusterType.Physical)
{
request.AddDataItem("");
}
if (OClient.ProtocolVersion >= 10)
{
request.AddDataItem("default");
}
else
{
request.AddDataItem((short)-1);
}
}
if (OClient.ProtocolVersion >= 18)
{
request.AddDataItem((short)-1); //clusterid
}
return(request);
}
public ClusterInfo(string name, ClusterType type, string location, string resourceGroup)
{
this.Name = name;
this.Type = type.ToString();
this.RpName = type == ClusterType.connectedClusters ? ArcClusterRP : AksClusterRP;
this.Location = location;
this.ResourceGroup = resourceGroup;
}
/// <summary>
/// Returns relevant details about the item in a string array.
/// </summary>
public string[] ToStringArray()
{
return(new[]
{
Name,
NodeAvailability.GetStartTime(),
NodeAvailability.GetAmPm(),
Zone,
Location.ToString(),
NodeType.ToString(),
ScripType.ToString(),
ReducesTo.ToString(),
ClusterType.ToString()
});
}
public static ClusterDetails CreateClusterIfNotExists(ClusterType clusterType)
{
var name = AppConfig.AzureResourceName + clusterType.ToString().ToLowerInvariant();
LOG.Info(String.Format("Creating Cluster: Name = {0}, Type = {1}", name, clusterType));
var parameters = new ClusterCreateParametersV2()
{
Name = name,
ClusterSizeInNodes = HDInsightClusterSize,
Location = AppConfig.AzureResourceLocation,
ClusterType = clusterType,
HeadNodeSize = "Large",
DataNodeSize = "Large",
ZookeeperNodeSize = "Medium",
UserName = AppConfig.AzureResourceUsername,
Password = AppConfig.AzureResourcePassword,
DefaultStorageAccountName = AzureStorageHelper.Name,
DefaultStorageAccountKey = AzureStorageHelper.PrimaryKey,
DefaultStorageContainer = AppConfig.AzureResourceName,
Version = "3.2",
//VirtualNetworkId = VirtualNetworkHelper.VNetId,
//SubnetName = VirtualNetworkHelper.SubnetName,
};
return CreateClusterIfNotExists(parameters);
}
public static void DeleteClusterIfExists(ClusterType clusterType)
{
var name = AppConfig.AzureResourceName + clusterType.ToString().ToLowerInvariant();
LOG.Info(String.Format("Deleting Cluster: Name = {0}, Type = {1}", name, clusterType));
DeleteClusterIfExists(name);
}
/// ------------------------------------------------------------------------------------
/// <summary>
/// Determine if the Current is the source.
/// </summary>
/// <param name="clusterType">Type of the cluster--should be a missing or added.</param>
/// <returns><c>true</c> if the cluster item(s) was/were added to current;
/// <c>false</c> if the cluster item(s) was/were missing in current</returns>
/// <exception cref="InvalidOperationException">thrown if clusterType is not added to current
/// or missing in current.</exception>
/// ------------------------------------------------------------------------------------
public static bool CurrentIsSource(ClusterType clusterType)
{
if (clusterType == ClusterType.AddedToCurrent || clusterType == ClusterType.OrphansInCurrent)
return true;
if (clusterType == ClusterType.MissingInCurrent || clusterType == ClusterType.OrphansInRevision)
return false;
throw new InvalidOperationException(string.Format(
"Cluster type {0} incompatible. Cluster type must be added to or missing in current.",
clusterType.ToString()));
}
internal void Cluster(string clusterName, ClusterType clustertype)
{
_compiler.Unique(Q.Cluster, ParseClassName(clusterName), clustertype.ToString().ToUpper());
}
static void Create(string subscriptionId, string resourceGroupName, string clusterDnsName, string clusterLocation,
List<AzureStorageConfig> asvAccounts, int clusterSize, string clusterUsername, string clusterPassword,
string hdInsightVersion, List<SqlAzureConfig> sqlAzureMetaStores,
ClusterType clusterType, OperatingSystemType osType)
{
Logger.InfoFormat("ResourceGroup: {0}, Cluster: {1} - Submitting a new cluster deployment request", resourceGroupName, clusterDnsName);
var clusterCreateParameters = new ClusterCreateParameters()
{
ClusterSizeInNodes = clusterSize,
ClusterType = (HDInsightClusterType)Enum.Parse(typeof(HDInsightClusterType), clusterType.ToString()),
DefaultStorageAccountKey = asvAccounts[0].Key,
DefaultStorageAccountName = asvAccounts[0].Name,
DefaultStorageContainer = asvAccounts[0].Container,
Location = clusterLocation,
Password = clusterPassword,
UserName = clusterUsername,
Version = hdInsightVersion,
OSType = (OSType)Enum.Parse(typeof(OSType), osType.ToString()),
};
if (clusterCreateParameters.OSType == OSType.Linux)
{
clusterCreateParameters.SshUserName = config.SshUsername;
if (String.IsNullOrWhiteSpace(config.SshPassword))
{
var publicKey = File.ReadAllText(config.SshPublicKeyFilePath);
Logger.Debug("SSH RSA Public Key: " + Environment.NewLine + publicKey + Environment.NewLine);
clusterCreateParameters.SshPublicKey = publicKey;
}
else
{
clusterCreateParameters.SshPassword = config.SshPassword;
}
}
else
{
if (config.AutoEnableRdp)
{
clusterCreateParameters.RdpUsername = config.RdpUsername;
clusterCreateParameters.RdpPassword = config.RdpPassword;
clusterCreateParameters.RdpAccessExpiry = DateTime.Now.AddDays(int.Parse(config.RdpExpirationInDays));
}
}
if (sqlAzureMetaStores != null && sqlAzureMetaStores.Count > 0)
{
var hiveMetastore = sqlAzureMetaStores.FirstOrDefault(s => s.Type.Equals("HiveMetastore"));
if (hiveMetastore != null)
{
clusterCreateParameters.HiveMetastore =
new Metastore(hiveMetastore.Server, hiveMetastore.Database, hiveMetastore.User, hiveMetastore.Password);
}
var oozieMetastore = sqlAzureMetaStores.FirstOrDefault(s => s.Type.Equals("OozieMetastore"));
if (oozieMetastore != null)
{
clusterCreateParameters.OozieMetastore =
new Metastore(oozieMetastore.Server, oozieMetastore.Database, oozieMetastore.User, oozieMetastore.Password);
}
}
var localStopWatch = Stopwatch.StartNew();
var createTask = hdInsightManagementClient.Clusters.CreateAsync(resourceGroupName, clusterDnsName, clusterCreateParameters);
Logger.InfoFormat("Cluster: {0} - Create cluster request submitted with task id: {1}, task status: {2}",
clusterDnsName, createTask.Id, createTask.Status);
Thread.Sleep(pollInterval);
var error = MonitorCreate(resourceGroupName, clusterDnsName, createTask);
if (error)
{
if (config.CleanupOnError)
{
Logger.InfoFormat("{0} - {1}. Submitting a delete request for the failed cluster creation.", Config.ConfigName.CleanupOnError.ToString(), config.CleanupOnError.ToString());
Delete(resourceGroupName, clusterDnsName);
}
else
{
throw new ApplicationException(String.Format("Cluster: {0} - Creation unsuccessful", clusterDnsName));
}
}
else
{
if (config.AutoEnableRdp && clusterCreateParameters.OSType == OSType.Windows)
{
HDInsightManagementCLIHelpers.CreateRdpFile(clusterDnsName, config.RdpUsername, config.RdpPassword);
}
}
}
