public void OnStart()
{
service.OnStart(this);
CountersReader counters = aeron.CountersReader();
int recoveryCounterId = AwaitRecoveryCounterId(counters);
isRecovering = true;
CheckForSnapshot(counters, recoveryCounterId);
CheckForReplay(counters, recoveryCounterId);
isRecovering = false;
JoinActiveLog(counters);
roleCounter = AwaitClusterRoleCounter(counters);
role = (ClusterRole)roleCounter.Get();
if (ClusterRole.Leader == role)
{
foreach (ClientSession session in sessionByIdMap.Values)
{
session.Connect(aeron);
}
}
}
private void Role(ClusterRole newRole)
{
if (newRole != role)
{
role = newRole;
service.OnRoleChange(newRole);
}
}
public ClusteringSpec RemoveRole(ClusterRole role)
{
var newValues = ImmutableList.ValueOf(Values.Where(value => value.Transform != role.Name));
if (newValues.Count == Values.Count)
{
return(this);
}
return(new ClusteringSpec(newValues));
}
private PassthroughResponse EnableRdp(ClusterRoleCollection roleCollection, string subscriptionId,
string cloudServiceName, string dnsName)
{
Assert.IsNotNull(roleCollection, "Role collection is null");
Assert.IsTrue(roleCollection.Count > 0, "There are no roles in the role collection");
var cluster = this.GetCluster(dnsName, cloudServiceName, subscriptionId);
if (cluster == null)
{
throw new ArgumentNullException(string.Format(ClustersTestConstants.ClusterDoesNotExistException, dnsName, subscriptionId));
}
ClusterRole previousRole = null;
foreach (var role in roleCollection)
{
Assert.IsNotNull(role, "The role is null");
Assert.IsTrue(role.RemoteDesktopSettings.IsEnabled);
if (previousRole != null)
{
Assert.IsTrue(
previousRole.RemoteDesktopSettings.AuthenticationCredential.Username ==
role.RemoteDesktopSettings.AuthenticationCredential.Username,
"rdpUsername between roles doesn't match");
Assert.IsTrue(
previousRole.RemoteDesktopSettings.AuthenticationCredential.Password ==
role.RemoteDesktopSettings.AuthenticationCredential.Password,
"rdpPassword between roles doesn't match");
Assert.IsTrue(previousRole.RemoteDesktopSettings.RemoteAccessExpiry ==
role.RemoteDesktopSettings.RemoteAccessExpiry,
"RemoteAccessExpory between roles doesn't match");
}
else
{
Assert.IsNotNull(role.RemoteDesktopSettings.AuthenticationCredential.Username,
"rdpUsername is null");
Assert.IsNotNull(role.RemoteDesktopSettings.AuthenticationCredential.Password,
"rdpPassword is null");
Assert.IsNotNull(role.RemoteDesktopSettings.RemoteAccessExpiry,
"RemoteAccessExpory is numm");
}
previousRole = role;
}
cluster.ClusterRoleCollection = roleCollection;
return(new PassthroughResponse {
Data = new Contracts.May2014.Operation {
OperationId = Guid.NewGuid().ToString(), Status = Contracts.May2014.OperationStatus.InProgress,
}
});
}
public static string GetDescription(this ClusterRole source)
{
string description;
try
{
FieldInfo fi = source.GetType().GetField(source.ToString());
var attributes = (DescriptionAttribute[])fi.GetCustomAttributes(typeof(DescriptionAttribute), false);
if (attributes.Length > 0)
{
return(attributes[0].Description);
}
description = source.ToString();
}
catch (Exception)
{
return("");
}
return(description);
}
public void OnRoleChange(ClusterRole newRole)
{
Console.WriteLine($"OnRoleChange: newRole={newRole}");
}
public static double?ToDouble(object value)
{
return(ClusterRole.ToDouble(value));
}
public static ClusteringSpec GetDefaultClusteringSpec(CancellationToken cancellationToken, ReportResults reportResults,
PivotedProperties pivotedProperties)
{
var values = new List <ValueSpec>();
foreach (var seriesGroup in pivotedProperties.SeriesGroups)
{
foreach (var series in seriesGroup.SeriesList)
{
var columnRef = ColumnRef.FromPivotedPropertySeries(series);
if (columnRef == null)
{
continue;
}
if (reportResults.RowCount >= MIN_ROWS_TO_ASSUME_HEADER && EqualValuesInAllRows(cancellationToken, reportResults, series))
{
values.Add(new ValueSpec(columnRef, ClusterRole.COLUMNHEADER));
}
else
{
var transform = ClusterRole.IsNumericType(series.PropertyType)
? ClusterRole.ZSCORE
: ClusterRole.BOOLEAN;
values.Add(new ValueSpec(columnRef, transform));
}
}
}
using (var propertyEnumerator = pivotedProperties.UngroupedProperties.GetEnumerator())
{
while (propertyEnumerator.MoveNext())
{
var columnRef = ColumnRef.FromPropertyDescriptor(propertyEnumerator.Current);
if (columnRef == null)
{
continue;
}
values.Insert(0, new ValueSpec(columnRef, ClusterRole.ROWHEADER));
break;
}
if (values.Count == 1)
{
while (propertyEnumerator.MoveNext())
{
var propertyDescriptor = propertyEnumerator.Current;
if (!ClusterRole.IsNumericType(propertyDescriptor?.PropertyType))
{
continue;
}
var columnRef = ColumnRef.FromPropertyDescriptor(propertyDescriptor);
if (columnRef == null)
{
continue;
}
values.Add(new ValueSpec(columnRef, ClusterRole.RAW));
}
if (values.Count == 1)
{
return(null);
}
}
}
if (values.Count == 0)
{
return(null);
}
return(new ClusteringSpec(values).ChangeDistanceMetric(ClusterMetricType.EUCLIDEAN.Name));
}
public ValueSpec ChangeTransform(ClusterRole transform)
{
return(ChangeTransform((transform ?? ClusterRole.IGNORED).Name));
}
public ValueSpec(ColumnRef columnRef, ClusterRole transform) : this(columnRef, transform.Name)
{
}
/// <summary>
/// Generate ClusterCreateParameters object for 2.X cluster with only Hadoop.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create2XClusterForMapReduceTemplate(HDInsight.ClusterCreateParametersV2 inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var cluster = new ClusterCreateParameters {
DnsName = inputs.Name, Version = inputs.Version
};
var remoteDesktopSettings = (string.IsNullOrEmpty(inputs.RdpUsername))
? new RemoteDesktopSettings()
{
IsEnabled = false
}
: new RemoteDesktopSettings()
{
IsEnabled = true,
AuthenticationCredential = new UsernamePasswordCredential()
{
Username = inputs.RdpUsername,
Password = inputs.RdpPassword
},
RemoteAccessExpiry = (DateTime)inputs.RdpAccessExpiry
};
var headnodeRole = new ClusterRole
{
FriendlyName = "HeadNodeRole",
InstanceCount = 2,
VMSizeAsString = inputs.HeadNodeSize,
RemoteDesktopSettings = remoteDesktopSettings
};
var workernodeRole = new ClusterRole
{
InstanceCount = inputs.ClusterSizeInNodes,
FriendlyName = "WorkerNodeRole",
VMSizeAsString = inputs.DataNodeSize,
RemoteDesktopSettings = remoteDesktopSettings
};
var zookeeperRole = new ClusterRole
{
InstanceCount = 3,
FriendlyName = "ZKRole",
VMSizeAsString = VmSize.Small.ToString(),
RemoteDesktopSettings = remoteDesktopSettings
};
cluster.ClusterRoleCollection.Add(headnodeRole);
cluster.ClusterRoleCollection.Add(workernodeRole);
cluster.ClusterRoleCollection.Add(zookeeperRole);
var gateway = new GatewayComponent
{
IsEnabled = true,
RestAuthCredential = new UsernamePasswordCredential {
Username = inputs.UserName, Password = inputs.Password
}
};
cluster.Components.Add(gateway);
cluster.Location = inputs.Location;
// Adding MapReduce component
MapReduceComponent mapReduce = new MapReduceComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
ConfigMapReduceComponent(mapReduce, inputs);
cluster.Components.Add(mapReduce);
// Adding Hive component
HiveComponent hive = new HiveComponent {
HeadNodeRole = headnodeRole
};
ConfigHiveComponent(hive, inputs);
cluster.Components.Add(hive);
// Adding config action component if needed
if (inputs.ConfigActions != null && inputs.ConfigActions.Count > 0)
{
CustomActionComponent configAction = new CustomActionComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
AddConfigActionComponent(configAction, inputs, headnodeRole, workernodeRole, zookeeperRole);
cluster.Components.Add(configAction);
}
// Adding Oozie component
OozieComponent oozie = new OozieComponent {
HeadNodeRole = headnodeRole
};
ConfigOozieComponent(oozie, inputs);
cluster.Components.Add(oozie);
// Adding Hdfs component
HdfsComponent hdfs = new HdfsComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
ConfigHdfsComponent(hdfs, inputs);
cluster.Components.Add(hdfs);
// Adding HadoopCore component
HadoopCoreComponent hadoopCore = new HadoopCoreComponent();
ConfigHadoopCoreComponent(hadoopCore, inputs);
cluster.Components.Add(hadoopCore);
ConfigVirtualNetwork(cluster, inputs);
return(cluster);
}
private static void AddConfigActionComponent(CustomActionComponent configAction, HDInsight.ClusterCreateParametersV2 inputs, ClusterRole headnodeRole, ClusterRole workernodeRole, ClusterRole zookeperRole)
{
configAction.CustomActions = new CustomActionList();
// Converts config action from PS/SDK to wire contract.
foreach (ConfigAction ca in inputs.ConfigActions)
{
CustomAction newConfigAction;
// Based on the config action type defined in SDK, convert them to config action defined in wire contract.
ScriptAction sca = ca as ScriptAction;
if (sca != null)
{
newConfigAction = new ScriptCustomAction
{
Name = ca.Name,
Uri = sca.Uri,
Parameters = sca.Parameters
};
}
else
{
throw new NotSupportedException("No such config action supported.");
}
newConfigAction.ClusterRoleCollection = new ClusterRoleCollection();
// Add in cluster role collection for each config action.
foreach (ClusterNodeType clusterRoleType in ca.ClusterRoleCollection)
{
if (clusterRoleType == ClusterNodeType.HeadNode)
{
newConfigAction.ClusterRoleCollection.Add(headnodeRole);
}
else if (clusterRoleType == ClusterNodeType.DataNode)
{
newConfigAction.ClusterRoleCollection.Add(workernodeRole);
}
else if (clusterRoleType == ClusterNodeType.ZookeperNode)
{
if (inputs.ClusterType.Equals(ClusterType.HBase) || inputs.ClusterType.Equals(ClusterType.Storm))
{
newConfigAction.ClusterRoleCollection.Add(zookeperRole);
}
else
{
throw new NotSupportedException(string.Format("Customization of zookeper nodes only supported for cluster types {0} and {1}",
ClusterType.HBase.ToString(), ClusterType.Storm.ToString()));
}
}
else
{
throw new NotSupportedException("No such node type supported.");
}
}
configAction.CustomActions.Add(newConfigAction);
}
}
/// <summary>
/// Generate ClusterCreateParameters object for 3.X cluster with only Hadoop.
/// </summary>
/// <param name="inputs">Cluster creation parameter inputs.</param>
/// <returns>The corresponding ClusterCreateParameter object.</returns>
internal static ClusterCreateParameters Create3XClusterFromMapReduceTemplate(HDInsight.ClusterCreateParametersV2 inputs)
{
if (inputs == null)
{
throw new ArgumentNullException("inputs");
}
var cluster = new ClusterCreateParameters
{
DnsName = inputs.Name,
Version = inputs.Version
};
var headnodeRole = new ClusterRole
{
FriendlyName = "HeadNodeRole",
InstanceCount = 2,
VMSizeAsString = inputs.HeadNodeSize,
};
var workernodeRole = new ClusterRole
{
InstanceCount = inputs.ClusterSizeInNodes,
FriendlyName = "WorkerNodeRole",
VMSizeAsString = inputs.DataNodeSize,
};
var zookeeperRole = new ClusterRole
{
InstanceCount = 3,
FriendlyName = "ZKRole",
VMSizeAsString = inputs.ZookeeperNodeSize ?? VmSize.Small.ToString(),
};
cluster.ClusterRoleCollection.Add(headnodeRole);
cluster.ClusterRoleCollection.Add(workernodeRole);
cluster.ClusterRoleCollection.Add(zookeeperRole);
var gateway = new GatewayComponent
{
IsEnabled = true,
RestAuthCredential = new UsernamePasswordCredential {
Username = inputs.UserName, Password = inputs.Password
}
};
cluster.Components.Add(gateway);
cluster.Location = inputs.Location;
//Add yarn component
YarnComponent yarn = new YarnComponent {
ResourceManagerRole = headnodeRole, NodeManagerRole = workernodeRole,
};
ConfigYarnComponent(yarn, inputs);
MapReduceApplication mapreduceApp = new MapReduceApplication();
ConfigMapReduceApplication(mapreduceApp, inputs);
yarn.Applications.Add(mapreduceApp);
cluster.Components.Add(yarn);
// Adding Hive component
HiveComponent hive = new HiveComponent {
HeadNodeRole = headnodeRole
};
ConfigHiveComponent(hive, inputs);
cluster.Components.Add(hive);
// Adding config action component if needed
if (inputs.ConfigActions != null && inputs.ConfigActions.Count > 0)
{
CustomActionComponent configAction = new CustomActionComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
AddConfigActionComponent(configAction, inputs, headnodeRole, workernodeRole);
cluster.Components.Add(configAction);
}
// Adding Oozie component
OozieComponent oozie = new OozieComponent {
HeadNodeRole = headnodeRole
};
ConfigOozieComponent(oozie, inputs);
cluster.Components.Add(oozie);
// Adding Hdfs component
HdfsComponent hdfs = new HdfsComponent {
HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
};
ConfigHdfsComponent(hdfs, inputs);
cluster.Components.Add(hdfs);
// Adding HadoopCore component
HadoopCoreComponent hadoopCore = new HadoopCoreComponent();
ConfigHadoopCoreComponent(hadoopCore, inputs);
cluster.Components.Add(hadoopCore);
// Adding Zookeeper component
cluster.Components.Add(new ZookeeperComponent {
ZookeeperRole = zookeeperRole
});
ConfigVirtualNetwork(cluster, inputs);
return(cluster);
}
public static ReportColorScheme FromClusteredResults(CancellationToken cancellationToken, ClusteredReportResults clusteredReportResults)
{
var discreteColorScheme = new DiscreteColorScheme();
var numericColorScheme = new NumericColorScheme();
var reportResults = clusteredReportResults;
var distinctValues = new HashSet <object>();
var numericValues = new HashSet <double>();
var colorManagers = new Dictionary <string, ColorManager>();
var columnHeaderValues = new Dictionary <string, object>();
var seriesColorManagers = new Dictionary <object, ColorManager>();
foreach (var property in reportResults.ClusteredProperties.RowHeaders)
{
cancellationToken.ThrowIfCancellationRequested();
if (ClusterRole.IsNumericType(property.PropertyType))
{
var colorScheme = new NumericColorScheme();
colorScheme.AddValues(reportResults.RowItems.Select(row => property.GetValue(row)));
colorManagers[property.Name] = new ColorManager(colorScheme, new[] { property }, null);
}
else
{
distinctValues.UnionWith(reportResults.RowItems.Select(property.GetValue).Where(v => null != v));
colorManagers[property.Name] = new ColorManager(discreteColorScheme, new[] { property }, null);
}
}
foreach (var property in reportResults.ClusteredProperties.PivotedProperties.UngroupedProperties)
{
cancellationToken.ThrowIfCancellationRequested();
var transform = reportResults.ClusteredProperties.GetRowTransform(property);
if (transform == null)
{
continue;
}
colorManagers[property.Name] = new ColorManager(numericColorScheme, new[] { property }, transform);
numericValues.UnionWith(reportResults.RowItems
.Select(row => transform.TransformRow(new[] { property.GetValue(row) }).First())
.OfType <double>());
}
foreach (var group in reportResults.PivotedProperties.SeriesGroups)
{
foreach (var series in group.SeriesList)
{
IColorScheme colorScheme = null;
var role = reportResults.ClusteredProperties.GetColumnRole(series);
if (role == ClusterRole.COLUMNHEADER)
{
var values = new HashSet <object>();
foreach (var property in series.PropertyDescriptors)
{
cancellationToken.ThrowIfCancellationRequested();
var columnDistinctValues = reportResults.RowItems.Select(property.GetValue)
.Where(v => null != v).Distinct().ToList();
if (columnDistinctValues.Count == 1)
{
columnHeaderValues[property.Name] = columnDistinctValues[0];
}
values.UnionWith(columnDistinctValues);
}
if (ClusterRole.IsNumericType(series.PropertyType))
{
colorScheme = new NumericColorScheme();
colorScheme.AddValues(values);
}
else
{
distinctValues.UnionWith(values);
colorScheme = discreteColorScheme;
}
}
else if (role is ClusterRole.Transform transform)
{
colorScheme = numericColorScheme;
numericValues.UnionWith(reportResults.RowItems.SelectMany(row =>
{
cancellationToken.ThrowIfCancellationRequested();
return(transform.TransformRow(
series.PropertyDescriptors.Select(pd => pd.GetValue(row))));
}).OfType <double>());
}
if (colorScheme != null)
{
var colorManager = new ColorManager(colorScheme, series.PropertyDescriptors, role as ClusterRole.Transform);
foreach (var pd in series.PropertyDescriptors)
{
colorManagers.Add(pd.Name, colorManager);
}
seriesColorManagers.Add(series.SeriesId, colorManager);
}
}
}
numericColorScheme.AddValues(numericValues);
discreteColorScheme.AddValues(distinctValues);
var columnColors = new Dictionary <string, Color>();
foreach (var entry in columnHeaderValues)
{
if (colorManagers.TryGetValue(entry.Key, out ColorManager manager))
{
var color = manager.ColorScheme.GetColor(entry.Value);
if (color.HasValue)
{
columnColors.Add(entry.Key, color.Value);
}
}
}
return(new ReportColorScheme(reportResults.ClusteredProperties, colorManagers, columnColors, seriesColorManagers));
}
private static void AddConfigActionComponent(CustomActionComponent configAction, HDInsight.ClusterCreateParameters inputs, ClusterRole headnodeRole, ClusterRole workernodeRole)
{
configAction.CustomActions = new CustomActionList();
// Converts config action from PS/SDK to wire contract.
foreach (ConfigAction ca in inputs.ConfigActions)
{
CustomAction newConfigAction;
// Based on the config action type defined in SDK, convert them to config action defined in wire contract.
ScriptAction sca = ca as ScriptAction;
if (sca != null)
{
newConfigAction = new ScriptCustomAction
{
Name = ca.Name,
Uri = sca.Uri,
Parameters = sca.Parameters
};
}
else
{
throw new NotSupportedException("No such config action supported.");
}
newConfigAction.ClusterRoleCollection = new ClusterRoleCollection();
// Add in cluster role collection for each config action.
foreach (ClusterNodeType clusterRoleType in ca.ClusterRoleCollection)
{
if (clusterRoleType == ClusterNodeType.HeadNode)
{
newConfigAction.ClusterRoleCollection.Add(headnodeRole);
}
else if (clusterRoleType == ClusterNodeType.DataNode)
{
newConfigAction.ClusterRoleCollection.Add(workernodeRole);
}
else
{
throw new NotSupportedException("No such node type supported.");
}
}
configAction.CustomActions.Add(newConfigAction);
}
}