示例#1
0
        public void OnStart()
        {
            service.OnStart(this);

            CountersReader counters          = aeron.CountersReader();
            int            recoveryCounterId = AwaitRecoveryCounterId(counters);

            isRecovering = true;
            CheckForSnapshot(counters, recoveryCounterId);
            CheckForReplay(counters, recoveryCounterId);
            isRecovering = false;

            JoinActiveLog(counters);

            roleCounter = AwaitClusterRoleCounter(counters);
            role        = (ClusterRole)roleCounter.Get();

            if (ClusterRole.Leader == role)
            {
                foreach (ClientSession session in sessionByIdMap.Values)
                {
                    session.Connect(aeron);
                }
            }
        }
示例#2
0
 private void Role(ClusterRole newRole)
 {
     if (newRole != role)
     {
         role = newRole;
         service.OnRoleChange(newRole);
     }
 }
示例#3
0
        public ClusteringSpec RemoveRole(ClusterRole role)
        {
            var newValues = ImmutableList.ValueOf(Values.Where(value => value.Transform != role.Name));

            if (newValues.Count == Values.Count)
            {
                return(this);
            }
            return(new ClusteringSpec(newValues));
        }
        private PassthroughResponse EnableRdp(ClusterRoleCollection roleCollection, string subscriptionId,
                                              string cloudServiceName, string dnsName)
        {
            Assert.IsNotNull(roleCollection, "Role collection is null");
            Assert.IsTrue(roleCollection.Count > 0, "There are no roles in the role collection");
            var cluster = this.GetCluster(dnsName, cloudServiceName, subscriptionId);

            if (cluster == null)
            {
                throw new ArgumentNullException(string.Format(ClustersTestConstants.ClusterDoesNotExistException, dnsName, subscriptionId));
            }
            ClusterRole previousRole = null;

            foreach (var role in roleCollection)
            {
                Assert.IsNotNull(role, "The role is null");
                Assert.IsTrue(role.RemoteDesktopSettings.IsEnabled);
                if (previousRole != null)
                {
                    Assert.IsTrue(
                        previousRole.RemoteDesktopSettings.AuthenticationCredential.Username ==
                        role.RemoteDesktopSettings.AuthenticationCredential.Username,
                        "rdpUsername between roles doesn't match");
                    Assert.IsTrue(
                        previousRole.RemoteDesktopSettings.AuthenticationCredential.Password ==
                        role.RemoteDesktopSettings.AuthenticationCredential.Password,
                        "rdpPassword between roles doesn't match");
                    Assert.IsTrue(previousRole.RemoteDesktopSettings.RemoteAccessExpiry ==
                                  role.RemoteDesktopSettings.RemoteAccessExpiry,
                                  "RemoteAccessExpory between roles doesn't match");
                }
                else
                {
                    Assert.IsNotNull(role.RemoteDesktopSettings.AuthenticationCredential.Username,
                                     "rdpUsername is null");
                    Assert.IsNotNull(role.RemoteDesktopSettings.AuthenticationCredential.Password,
                                     "rdpPassword is null");
                    Assert.IsNotNull(role.RemoteDesktopSettings.RemoteAccessExpiry,
                                     "RemoteAccessExpory is numm");
                }
                previousRole = role;
            }

            cluster.ClusterRoleCollection = roleCollection;
            return(new PassthroughResponse {
                Data = new Contracts.May2014.Operation {
                    OperationId = Guid.NewGuid().ToString(), Status = Contracts.May2014.OperationStatus.InProgress,
                }
            });
        }
        public static string GetDescription(this ClusterRole source)
        {
            string description;

            try
            {
                FieldInfo fi = source.GetType().GetField(source.ToString());

                var attributes = (DescriptionAttribute[])fi.GetCustomAttributes(typeof(DescriptionAttribute), false);

                if (attributes.Length > 0)
                {
                    return(attributes[0].Description);
                }

                description = source.ToString();
            }
            catch (Exception)
            {
                return("");
            }
            return(description);
        }
示例#6
0
 public void OnRoleChange(ClusterRole newRole)
 {
     Console.WriteLine($"OnRoleChange: newRole={newRole}");
 }
示例#7
0
 public static double?ToDouble(object value)
 {
     return(ClusterRole.ToDouble(value));
 }
示例#8
0
        public static ClusteringSpec GetDefaultClusteringSpec(CancellationToken cancellationToken, ReportResults reportResults,
                                                              PivotedProperties pivotedProperties)
        {
            var values = new List <ValueSpec>();

            foreach (var seriesGroup in pivotedProperties.SeriesGroups)
            {
                foreach (var series in seriesGroup.SeriesList)
                {
                    var columnRef = ColumnRef.FromPivotedPropertySeries(series);
                    if (columnRef == null)
                    {
                        continue;
                    }
                    if (reportResults.RowCount >= MIN_ROWS_TO_ASSUME_HEADER && EqualValuesInAllRows(cancellationToken, reportResults, series))
                    {
                        values.Add(new ValueSpec(columnRef, ClusterRole.COLUMNHEADER));
                    }
                    else
                    {
                        var transform = ClusterRole.IsNumericType(series.PropertyType)
                            ? ClusterRole.ZSCORE
                            : ClusterRole.BOOLEAN;
                        values.Add(new ValueSpec(columnRef, transform));
                    }
                }
            }

            using (var propertyEnumerator = pivotedProperties.UngroupedProperties.GetEnumerator())
            {
                while (propertyEnumerator.MoveNext())
                {
                    var columnRef = ColumnRef.FromPropertyDescriptor(propertyEnumerator.Current);
                    if (columnRef == null)
                    {
                        continue;
                    }
                    values.Insert(0, new ValueSpec(columnRef, ClusterRole.ROWHEADER));
                    break;
                }

                if (values.Count == 1)
                {
                    while (propertyEnumerator.MoveNext())
                    {
                        var propertyDescriptor = propertyEnumerator.Current;
                        if (!ClusterRole.IsNumericType(propertyDescriptor?.PropertyType))
                        {
                            continue;
                        }

                        var columnRef = ColumnRef.FromPropertyDescriptor(propertyDescriptor);
                        if (columnRef == null)
                        {
                            continue;
                        }
                        values.Add(new ValueSpec(columnRef, ClusterRole.RAW));
                    }

                    if (values.Count == 1)
                    {
                        return(null);
                    }
                }
            }

            if (values.Count == 0)
            {
                return(null);
            }
            return(new ClusteringSpec(values).ChangeDistanceMetric(ClusterMetricType.EUCLIDEAN.Name));
        }
示例#9
0
 public ValueSpec ChangeTransform(ClusterRole transform)
 {
     return(ChangeTransform((transform ?? ClusterRole.IGNORED).Name));
 }
示例#10
0
 public ValueSpec(ColumnRef columnRef, ClusterRole transform) : this(columnRef, transform.Name)
 {
 }
        /// <summary>
        /// Generate ClusterCreateParameters object for 2.X cluster with only Hadoop.
        /// </summary>
        /// <param name="inputs">Cluster creation parameter inputs.</param>
        /// <returns>The corresponding ClusterCreateParameter object.</returns>
        internal static ClusterCreateParameters Create2XClusterForMapReduceTemplate(HDInsight.ClusterCreateParametersV2 inputs)
        {
            if (inputs == null)
            {
                throw new ArgumentNullException("inputs");
            }

            var cluster = new ClusterCreateParameters {
                DnsName = inputs.Name, Version = inputs.Version
            };
            var remoteDesktopSettings = (string.IsNullOrEmpty(inputs.RdpUsername))
                ? new RemoteDesktopSettings()
            {
                IsEnabled = false
            }
                : new RemoteDesktopSettings()
            {
                IsEnabled = true,
                AuthenticationCredential = new UsernamePasswordCredential()
                {
                    Username = inputs.RdpUsername,
                    Password = inputs.RdpPassword
                },
                RemoteAccessExpiry = (DateTime)inputs.RdpAccessExpiry
            };
            var headnodeRole = new ClusterRole
            {
                FriendlyName          = "HeadNodeRole",
                InstanceCount         = 2,
                VMSizeAsString        = inputs.HeadNodeSize,
                RemoteDesktopSettings = remoteDesktopSettings
            };
            var workernodeRole = new ClusterRole
            {
                InstanceCount         = inputs.ClusterSizeInNodes,
                FriendlyName          = "WorkerNodeRole",
                VMSizeAsString        = inputs.DataNodeSize,
                RemoteDesktopSettings = remoteDesktopSettings
            };
            var zookeeperRole = new ClusterRole
            {
                InstanceCount         = 3,
                FriendlyName          = "ZKRole",
                VMSizeAsString        = VmSize.Small.ToString(),
                RemoteDesktopSettings = remoteDesktopSettings
            };

            cluster.ClusterRoleCollection.Add(headnodeRole);
            cluster.ClusterRoleCollection.Add(workernodeRole);
            cluster.ClusterRoleCollection.Add(zookeeperRole);

            var gateway = new GatewayComponent
            {
                IsEnabled          = true,
                RestAuthCredential = new UsernamePasswordCredential {
                    Username = inputs.UserName, Password = inputs.Password
                }
            };

            cluster.Components.Add(gateway);
            cluster.Location = inputs.Location;

            // Adding MapReduce component
            MapReduceComponent mapReduce = new MapReduceComponent {
                HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
            };

            ConfigMapReduceComponent(mapReduce, inputs);
            cluster.Components.Add(mapReduce);

            // Adding Hive component
            HiveComponent hive = new HiveComponent {
                HeadNodeRole = headnodeRole
            };

            ConfigHiveComponent(hive, inputs);
            cluster.Components.Add(hive);

            // Adding config action component if needed
            if (inputs.ConfigActions != null && inputs.ConfigActions.Count > 0)
            {
                CustomActionComponent configAction = new CustomActionComponent {
                    HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
                };
                AddConfigActionComponent(configAction, inputs, headnodeRole, workernodeRole, zookeeperRole);
                cluster.Components.Add(configAction);
            }

            // Adding Oozie component
            OozieComponent oozie = new OozieComponent {
                HeadNodeRole = headnodeRole
            };

            ConfigOozieComponent(oozie, inputs);
            cluster.Components.Add(oozie);

            // Adding Hdfs component
            HdfsComponent hdfs = new HdfsComponent {
                HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
            };

            ConfigHdfsComponent(hdfs, inputs);
            cluster.Components.Add(hdfs);

            // Adding HadoopCore component
            HadoopCoreComponent hadoopCore = new HadoopCoreComponent();

            ConfigHadoopCoreComponent(hadoopCore, inputs);
            cluster.Components.Add(hadoopCore);

            ConfigVirtualNetwork(cluster, inputs);

            return(cluster);
        }
        private static void AddConfigActionComponent(CustomActionComponent configAction, HDInsight.ClusterCreateParametersV2 inputs, ClusterRole headnodeRole, ClusterRole workernodeRole, ClusterRole zookeperRole)
        {
            configAction.CustomActions = new CustomActionList();

            // Converts config action from PS/SDK to wire contract.
            foreach (ConfigAction ca in inputs.ConfigActions)
            {
                CustomAction newConfigAction;

                // Based on the config action type defined in SDK, convert them to config action defined in wire contract.
                ScriptAction sca = ca as ScriptAction;

                if (sca != null)
                {
                    newConfigAction = new ScriptCustomAction
                    {
                        Name       = ca.Name,
                        Uri        = sca.Uri,
                        Parameters = sca.Parameters
                    };
                }
                else
                {
                    throw new NotSupportedException("No such config action supported.");
                }

                newConfigAction.ClusterRoleCollection = new ClusterRoleCollection();

                // Add in cluster role collection for each config action.
                foreach (ClusterNodeType clusterRoleType in ca.ClusterRoleCollection)
                {
                    if (clusterRoleType == ClusterNodeType.HeadNode)
                    {
                        newConfigAction.ClusterRoleCollection.Add(headnodeRole);
                    }
                    else if (clusterRoleType == ClusterNodeType.DataNode)
                    {
                        newConfigAction.ClusterRoleCollection.Add(workernodeRole);
                    }
                    else if (clusterRoleType == ClusterNodeType.ZookeperNode)
                    {
                        if (inputs.ClusterType.Equals(ClusterType.HBase) || inputs.ClusterType.Equals(ClusterType.Storm))
                        {
                            newConfigAction.ClusterRoleCollection.Add(zookeperRole);
                        }
                        else
                        {
                            throw new NotSupportedException(string.Format("Customization of zookeper nodes only supported for cluster types {0} and {1}",
                                                                          ClusterType.HBase.ToString(), ClusterType.Storm.ToString()));
                        }
                    }
                    else
                    {
                        throw new NotSupportedException("No such node type supported.");
                    }
                }

                configAction.CustomActions.Add(newConfigAction);
            }
        }
        /// <summary>
        /// Generate ClusterCreateParameters object for 3.X cluster with only Hadoop.
        /// </summary>
        /// <param name="inputs">Cluster creation parameter inputs.</param>
        /// <returns>The corresponding ClusterCreateParameter object.</returns>
        internal static ClusterCreateParameters Create3XClusterFromMapReduceTemplate(HDInsight.ClusterCreateParametersV2 inputs)
        {
            if (inputs == null)
            {
                throw new ArgumentNullException("inputs");
            }

            var cluster = new ClusterCreateParameters
            {
                DnsName = inputs.Name,
                Version = inputs.Version
            };
            var headnodeRole = new ClusterRole
            {
                FriendlyName   = "HeadNodeRole",
                InstanceCount  = 2,
                VMSizeAsString = inputs.HeadNodeSize,
            };
            var workernodeRole = new ClusterRole
            {
                InstanceCount  = inputs.ClusterSizeInNodes,
                FriendlyName   = "WorkerNodeRole",
                VMSizeAsString = inputs.DataNodeSize,
            };
            var zookeeperRole = new ClusterRole
            {
                InstanceCount  = 3,
                FriendlyName   = "ZKRole",
                VMSizeAsString = inputs.ZookeeperNodeSize ?? VmSize.Small.ToString(),
            };

            cluster.ClusterRoleCollection.Add(headnodeRole);
            cluster.ClusterRoleCollection.Add(workernodeRole);
            cluster.ClusterRoleCollection.Add(zookeeperRole);

            var gateway = new GatewayComponent
            {
                IsEnabled          = true,
                RestAuthCredential = new UsernamePasswordCredential {
                    Username = inputs.UserName, Password = inputs.Password
                }
            };

            cluster.Components.Add(gateway);
            cluster.Location = inputs.Location;

            //Add yarn component
            YarnComponent yarn = new YarnComponent {
                ResourceManagerRole = headnodeRole, NodeManagerRole = workernodeRole,
            };

            ConfigYarnComponent(yarn, inputs);
            MapReduceApplication mapreduceApp = new MapReduceApplication();

            ConfigMapReduceApplication(mapreduceApp, inputs);
            yarn.Applications.Add(mapreduceApp);
            cluster.Components.Add(yarn);

            // Adding Hive component
            HiveComponent hive = new HiveComponent {
                HeadNodeRole = headnodeRole
            };

            ConfigHiveComponent(hive, inputs);
            cluster.Components.Add(hive);

            // Adding config action component if needed
            if (inputs.ConfigActions != null && inputs.ConfigActions.Count > 0)
            {
                CustomActionComponent configAction = new CustomActionComponent {
                    HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
                };
                AddConfigActionComponent(configAction, inputs, headnodeRole, workernodeRole);
                cluster.Components.Add(configAction);
            }

            // Adding Oozie component
            OozieComponent oozie = new OozieComponent {
                HeadNodeRole = headnodeRole
            };

            ConfigOozieComponent(oozie, inputs);
            cluster.Components.Add(oozie);

            // Adding Hdfs component
            HdfsComponent hdfs = new HdfsComponent {
                HeadNodeRole = headnodeRole, WorkerNodeRole = workernodeRole
            };

            ConfigHdfsComponent(hdfs, inputs);
            cluster.Components.Add(hdfs);

            // Adding HadoopCore component
            HadoopCoreComponent hadoopCore = new HadoopCoreComponent();

            ConfigHadoopCoreComponent(hadoopCore, inputs);
            cluster.Components.Add(hadoopCore);

            // Adding Zookeeper component
            cluster.Components.Add(new ZookeeperComponent {
                ZookeeperRole = zookeeperRole
            });

            ConfigVirtualNetwork(cluster, inputs);

            return(cluster);
        }
示例#14
0
        public static ReportColorScheme FromClusteredResults(CancellationToken cancellationToken, ClusteredReportResults clusteredReportResults)
        {
            var discreteColorScheme = new DiscreteColorScheme();
            var numericColorScheme  = new NumericColorScheme();
            var reportResults       = clusteredReportResults;
            var distinctValues      = new HashSet <object>();
            var numericValues       = new HashSet <double>();
            var colorManagers       = new Dictionary <string, ColorManager>();
            var columnHeaderValues  = new Dictionary <string, object>();
            var seriesColorManagers = new Dictionary <object, ColorManager>();

            foreach (var property in reportResults.ClusteredProperties.RowHeaders)
            {
                cancellationToken.ThrowIfCancellationRequested();
                if (ClusterRole.IsNumericType(property.PropertyType))
                {
                    var colorScheme = new NumericColorScheme();
                    colorScheme.AddValues(reportResults.RowItems.Select(row => property.GetValue(row)));
                    colorManagers[property.Name] = new ColorManager(colorScheme, new[] { property }, null);
                }
                else
                {
                    distinctValues.UnionWith(reportResults.RowItems.Select(property.GetValue).Where(v => null != v));
                    colorManagers[property.Name] = new ColorManager(discreteColorScheme, new[] { property }, null);
                }
            }
            foreach (var property in reportResults.ClusteredProperties.PivotedProperties.UngroupedProperties)
            {
                cancellationToken.ThrowIfCancellationRequested();
                var transform = reportResults.ClusteredProperties.GetRowTransform(property);
                if (transform == null)
                {
                    continue;
                }
                colorManagers[property.Name] = new ColorManager(numericColorScheme, new[] { property }, transform);
                numericValues.UnionWith(reportResults.RowItems
                                        .Select(row => transform.TransformRow(new[] { property.GetValue(row) }).First())
                                        .OfType <double>());
            }

            foreach (var group in reportResults.PivotedProperties.SeriesGroups)
            {
                foreach (var series in group.SeriesList)
                {
                    IColorScheme colorScheme = null;
                    var          role        = reportResults.ClusteredProperties.GetColumnRole(series);
                    if (role == ClusterRole.COLUMNHEADER)
                    {
                        var values = new HashSet <object>();
                        foreach (var property in series.PropertyDescriptors)
                        {
                            cancellationToken.ThrowIfCancellationRequested();
                            var columnDistinctValues = reportResults.RowItems.Select(property.GetValue)
                                                       .Where(v => null != v).Distinct().ToList();
                            if (columnDistinctValues.Count == 1)
                            {
                                columnHeaderValues[property.Name] = columnDistinctValues[0];
                            }
                            values.UnionWith(columnDistinctValues);
                        }
                        if (ClusterRole.IsNumericType(series.PropertyType))
                        {
                            colorScheme = new NumericColorScheme();
                            colorScheme.AddValues(values);
                        }
                        else
                        {
                            distinctValues.UnionWith(values);
                            colorScheme = discreteColorScheme;
                        }
                    }
                    else if (role is ClusterRole.Transform transform)
                    {
                        colorScheme = numericColorScheme;
                        numericValues.UnionWith(reportResults.RowItems.SelectMany(row =>
                        {
                            cancellationToken.ThrowIfCancellationRequested();
                            return(transform.TransformRow(
                                       series.PropertyDescriptors.Select(pd => pd.GetValue(row))));
                        }).OfType <double>());
                    }

                    if (colorScheme != null)
                    {
                        var colorManager = new ColorManager(colorScheme, series.PropertyDescriptors, role as ClusterRole.Transform);
                        foreach (var pd in series.PropertyDescriptors)
                        {
                            colorManagers.Add(pd.Name, colorManager);
                        }
                        seriesColorManagers.Add(series.SeriesId, colorManager);
                    }
                }
            }
            numericColorScheme.AddValues(numericValues);
            discreteColorScheme.AddValues(distinctValues);
            var columnColors = new Dictionary <string, Color>();

            foreach (var entry in columnHeaderValues)
            {
                if (colorManagers.TryGetValue(entry.Key, out ColorManager manager))
                {
                    var color = manager.ColorScheme.GetColor(entry.Value);
                    if (color.HasValue)
                    {
                        columnColors.Add(entry.Key, color.Value);
                    }
                }
            }
            return(new ReportColorScheme(reportResults.ClusteredProperties, colorManagers, columnColors, seriesColorManagers));
        }
        private static void AddConfigActionComponent(CustomActionComponent configAction, HDInsight.ClusterCreateParameters inputs, ClusterRole headnodeRole, ClusterRole workernodeRole)
        {
            configAction.CustomActions = new CustomActionList();

            // Converts config action from PS/SDK to wire contract.
            foreach (ConfigAction ca in inputs.ConfigActions)
            {
                CustomAction newConfigAction;

                // Based on the config action type defined in SDK, convert them to config action defined in wire contract.
                ScriptAction sca = ca as ScriptAction;

                if (sca != null)
                {
                    newConfigAction = new ScriptCustomAction
                    {
                        Name       = ca.Name,
                        Uri        = sca.Uri,
                        Parameters = sca.Parameters
                    };
                }
                else
                {
                    throw new NotSupportedException("No such config action supported.");
                }

                newConfigAction.ClusterRoleCollection = new ClusterRoleCollection();

                // Add in cluster role collection for each config action.
                foreach (ClusterNodeType clusterRoleType in ca.ClusterRoleCollection)
                {
                    if (clusterRoleType == ClusterNodeType.HeadNode)
                    {
                        newConfigAction.ClusterRoleCollection.Add(headnodeRole);
                    }
                    else if (clusterRoleType == ClusterNodeType.DataNode)
                    {
                        newConfigAction.ClusterRoleCollection.Add(workernodeRole);
                    }
                    else
                    {
                        throw new NotSupportedException("No such node type supported.");
                    }
                }

                configAction.CustomActions.Add(newConfigAction);
            }
        }