public static ClusterCreateParametersExtended GetIaasClusterSpec()
{
var cluster = new ClusterCreateParametersExtended
{
Location = "West US",
Properties = new ClusterCreateProperties
{
ClusterDefinition = new ClusterDefinition
{
ClusterType = "Hadoop"
},
ClusterVersion = "3.2",
OperatingSystemType = OSType.Linux
}
};
var coreConfigs = new Dictionary<string, string>
{
{"fs.defaultFS", string.Format("wasb://{0}@{1}", DefaultContainer, StorageAccountName)},
{
string.Format("fs.azure.account.key.{0}", StorageAccountName),
StorageAccountKey
}
};
var gatewayConfigs = new Dictionary<string, string>
{
{"restAuthCredential.isEnabled", "true"},
{"restAuthCredential.username", HttpUser},
{"restAuthCredential.password", HttpPassword}
};
var configurations = new Dictionary<string, Dictionary<string, string>>
{
{"core-site", coreConfigs},
{"gateway", gatewayConfigs}
};
var serializedConfig = JsonConvert.SerializeObject(configurations);
cluster.Properties.ClusterDefinition.Configurations = serializedConfig;
cluster.Tags.Add("tag1", "value1");
cluster.Tags.Add("tag2", "value2");
var sshPublicKeys = new List<SshPublicKey>();
var sshPublicKey = new SshPublicKey
{
CertificateData =
string.Format("ssh-rsa {0}", SshKey)
};
sshPublicKeys.Add(sshPublicKey);
var headNode = new Role
{
Name = "headnode",
TargetInstanceCount = 1,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
UserName = "******",
SshProfile = new SshProfile
{
SshPublicKeys = sshPublicKeys
}
}
},
VirtualNetworkProfile = new VirtualNetworkProfile
{
Id = "vnetid",
SubnetName = "subnetname"
}
};
var workerNode = new Role
{
Name = "workernode",
TargetInstanceCount = 1,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
},
OsProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
UserName = "******",
SshProfile = new SshProfile
{
SshPublicKeys = sshPublicKeys
}
}
}
};
cluster.Properties.ComputeProfile = new ComputeProfile();
cluster.Properties.ComputeProfile.Roles.Add(headNode);
cluster.Properties.ComputeProfile.Roles.Add(workerNode);
return cluster;
}
private static IEnumerable<Role> GetRoleCollection(ClusterCreateParameters clusterCreateParameters)
{
//OS Profile
var osProfile = new OsProfile();
if (clusterCreateParameters.OSType == OSType.Windows)
{
RdpSettings rdpSettings = null;
if (!string.IsNullOrEmpty(clusterCreateParameters.RdpUsername))
{
rdpSettings = new RdpSettings
{
UserName = clusterCreateParameters.RdpUsername,
Password = clusterCreateParameters.RdpPassword,
ExpiryDate = clusterCreateParameters.RdpAccessExpiry
};
}
osProfile = new OsProfile
{
WindowsOperatingSystemProfile = new WindowsOperatingSystemProfile
{
RdpSettings = rdpSettings
}
};
}
else if (clusterCreateParameters.OSType == OSType.Linux)
{
var sshPublicKeys = new List<SshPublicKey>();
if (!string.IsNullOrEmpty(clusterCreateParameters.SshPublicKey))
{
var sshPublicKey = new SshPublicKey
{
CertificateData = clusterCreateParameters.SshPublicKey
};
sshPublicKeys.Add(sshPublicKey);
}
SshProfile sshProfile;
if (sshPublicKeys.Count > 0)
{
sshProfile = new SshProfile
{
SshPublicKeys = sshPublicKeys
};
}
else
{
sshProfile = null;
}
osProfile = new OsProfile
{
LinuxOperatingSystemProfile = new LinuxOperatingSystemProfile
{
UserName = clusterCreateParameters.SshUserName,
Password = clusterCreateParameters.SshPassword,
SshProfile = sshProfile
}
};
}
//VNet Profile
var vnetProfile = new VirtualNetworkProfile();
if (!string.IsNullOrEmpty(clusterCreateParameters.VirtualNetworkId))
{
vnetProfile.Id = clusterCreateParameters.VirtualNetworkId;
}
if (!string.IsNullOrEmpty(clusterCreateParameters.SubnetName))
{
vnetProfile.SubnetName = clusterCreateParameters.SubnetName;
}
if (string.IsNullOrEmpty(vnetProfile.Id) && string.IsNullOrEmpty(vnetProfile.SubnetName))
{
vnetProfile = null;
}
List<ScriptAction> workernodeactions = null;
List<ScriptAction> headnodeactions = null;
List<ScriptAction> zookeepernodeactions = null;
//Script Actions
foreach (var scriptAction in clusterCreateParameters.ScriptActions)
{
if (scriptAction.Key.ToString().ToLower().Equals("workernode"))
{
workernodeactions = scriptAction.Value;
}
else if (scriptAction.Key.ToString().ToLower().Equals("headnode"))
{
headnodeactions = scriptAction.Value;
}
else if (scriptAction.Key.ToString().ToLower().Equals("zookeepernode"))
{
zookeepernodeactions = scriptAction.Value;
}
}
//Roles
var roles = new List<Role>();
var headNodeSize = GetHeadNodeSize(clusterCreateParameters);
var headNode = new Role
{
Name = "headnode",
TargetInstanceCount = 2,
HardwareProfile = new HardwareProfile
{
VmSize = headNodeSize
},
OsProfile = osProfile,
VirtualNetworkProfile = vnetProfile,
ScriptActions = headnodeactions
};
roles.Add(headNode);
var workerNodeSize = GetWorkerNodeSize(clusterCreateParameters);
var workerNode = new Role
{
Name = "workernode",
TargetInstanceCount = clusterCreateParameters.ClusterSizeInNodes,
HardwareProfile = new HardwareProfile
{
VmSize = workerNodeSize
},
OsProfile = osProfile,
ScriptActions = workernodeactions
};
roles.Add(workerNode);
if (clusterCreateParameters.OSType == OSType.Windows)
{
if (clusterCreateParameters.ClusterType == HDInsightClusterType.Hadoop ||
clusterCreateParameters.ClusterType == HDInsightClusterType.Spark)
{
return roles;
}
}
if (clusterCreateParameters.OSType == OSType.Linux)
{
if (clusterCreateParameters.ClusterType == HDInsightClusterType.Hadoop ||
clusterCreateParameters.ClusterType == HDInsightClusterType.Spark)
{
clusterCreateParameters.ZookeeperNodeSize = "Small";
}
}
string zookeeperNodeSize = clusterCreateParameters.ZookeeperNodeSize ?? "Medium";
var zookeepernode = new Role
{
Name = "zookeepernode",
ScriptActions = zookeepernodeactions,
TargetInstanceCount = 3,
OsProfile = osProfile,
HardwareProfile = new HardwareProfile
{
VmSize = zookeeperNodeSize
}
};
roles.Add(zookeepernode);
return roles;
}
public static ClusterCreateParametersExtended GetPaasClusterSpec()
{
var cluster = new ClusterCreateParametersExtended
{
Location = "West US",
Properties = new ClusterCreateProperties
{
ClusterDefinition = new ClusterDefinition
{
ClusterType = "Hadoop"
},
ClusterVersion = "3.1",
OperatingSystemType = OSType.Windows
}
};
var coreConfigs = new Dictionary<string, string>
{
{"fs.defaultFS", string.Format("wasb://{0}@{1}", DefaultContainer, StorageAccountName)},
{
string.Format("fs.azure.account.key.{0}", StorageAccountName), StorageAccountKey
}
};
var gatewayConfigs = new Dictionary<string, string>
{
{"restAuthCredential.isEnabled", "true"},
{"restAuthCredential.username", HttpUser},
{"restAuthCredential.password", HttpPassword}
};
var configurations = new Dictionary<string, Dictionary<string, string>>
{
{"core-site", coreConfigs},
{"gateway", gatewayConfigs}
};
var serializedConfig = JsonConvert.SerializeObject(configurations);
cluster.Properties.ClusterDefinition.Configurations = serializedConfig;
cluster.Tags.Add("tag1", "value1");
cluster.Tags.Add("tag2", "value2");
var headNode = new Role
{
Name = "headnode",
TargetInstanceCount = 2,
HardwareProfile = new HardwareProfile
{
VmSize = "ExtraLarge"
},
OsProfile = new OsProfile
{
WindowsOperatingSystemProfile = new WindowsOperatingSystemProfile
{
RdpSettings = new RdpSettings
{
UserName = RdpUser,
Password = RdpPassword,
ExpiryDate = new DateTime(2025, 3, 1)
}
}
}
};
var workerNode = new Role
{
Name = "workernode",
TargetInstanceCount = 5,
HardwareProfile = new HardwareProfile
{
VmSize = "Large"
}
};
cluster.Properties.ComputeProfile = new ComputeProfile();
cluster.Properties.ComputeProfile.Roles.Add(headNode);
cluster.Properties.ComputeProfile.Roles.Add(workerNode);
return cluster;
}