protected ClusterBase(IProgress <double> progress, CancellationTokenSource cts, ClusterOptions clusterOptions, ILogger logger, ILoggerFactory loggerFactory) { Logger = logger; Logger.LogInformation(@" # ______ __ ______ __ __ ____ _ __ __ # / ____/________ _____ ____/ / / ____/__ ____ / /__________ _/ / / __ \(_)________ ____ _/ /______/ /_ # / / __/ ___/ __ `/ __ \/ __ / / / / _ \/ __ \/ __/ ___/ __ `/ / / / / / / ___/ __ \/ __ `/ __/ ___/ __ \ # / /_/ / / / /_/ / / / / /_/ / / /___/ __/ / / / /_/ / / /_/ / / / /_/ / (__ ) /_/ / /_/ / /_/ /__/ / / / # \____/_/ \__,_/_/ /_/\__,_/ \____/\___/_/ /_/\__/_/ \__,_/_/ /_____/_/____/ .___/\__,_/\__/\___/_/ /_/ # /_/ "); ClusterOptions = clusterOptions; Progress = progress ?? new Progress <double>(); CancellationTokenSource = cts ?? new CancellationTokenSource(); var policyResult = Policy.Handle <Exception>().RetryAsync().ExecuteAndCaptureAsync(async() => { PersistentCache = new CachingService(new PersistentCacheProvider( await SQLiteDatabase.Connection, new LazyCache.CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions { SizeLimit = ClusterOptions.MaxItemsInPersistentCache }))), loggerFactory)); }).GetAwaiter().GetResult(); if (policyResult.Outcome == OutcomeType.Failure) { PersistentCache = new CachingService(new DummyCacheProvider()); Logger.LogError( $"Error while creating persistence cache: {policyResult.FinalException?.Message ?? string.Empty}."); } ClusterMetrics = new ClusterMetrics(Guid.NewGuid()); var interval = new Subject <Unit>(); var scheduler = Scheduler.Default; _computeClusterHealthSubscription = interval.Select(_ => Observable.Interval(TimeSpan.FromSeconds(5))) .Switch() .ObserveOn(scheduler).Subscribe(e => ComputeClusterHealth()); interval.OnNext(Unit.Default); _counterMonitor.CounterUpdate += OnCounterUpdate; var monitorTask = new Task(() => { try { _counterMonitor.Start(Logger); } catch (Exception ex) { Logger.LogError($"Error while listening to counters: {ex.Message}"); } }); monitorTask.Start(); }
/// <summary>Choose a Machine in runtime according to the cluster status.</summary> /// <exception cref="System.IO.IOException"/> private DistSum.Machine ChooseMachine(Configuration conf) { int parts = conf.GetInt(NParts, int.MaxValue); try { for (; ; Sharpen.Thread.Sleep(2000)) { //get cluster status ClusterMetrics status = cluster.GetClusterStatus(); int m = status.GetMapSlotCapacity() - status.GetOccupiedMapSlots(); int r = status.GetReduceSlotCapacity() - status.GetOccupiedReduceSlots(); if (m >= parts || r >= parts) { //favor ReduceSide machine DistSum.Machine value = r >= parts ? DistSum.ReduceSide.Instance : DistSum.MapSide .Instance; [email protected](" " + this + " is " + value + " (m=" + m + ", r=" + r + ")"); return(value); } } } catch (Exception e) { throw new IOException(e); } }
public async Task TestGetClusterMetrics() { var ctx = new TestContext(); var urlProvider = ctx.UrlProviderFake; var restReqExecutor = ctx.RestRequestExecutorFake; var anyUri = Enumerable.Repeat(new Uri("anyscheme://anypath"), 1); urlProvider.GetUrlAsync().Returns(Task.FromResult(anyUri)); var anyClusterMetrics = new ClusterMetrics { ActiveNodes = 5, AllocatedMB = 1000, AllocatedVirtualCores = 10, AppsCompleted = 301 }; restReqExecutor.ExecuteAsync <ClusterMetrics>( Arg.Is <RestRequest>( req => req.Resource == "ws/v1/cluster/metrics" && req.RootElement == "clusterMetrics" && req.Method == Method.GET), anyUri.First(), CancellationToken.None).Returns(Task.FromResult(anyClusterMetrics)); var yarnClient = ctx.GetClient(); ClusterMetrics actualClusterMetrics = await yarnClient.GetClusterMetricsAsync(); Assert.Equal(anyClusterMetrics, actualClusterMetrics); var unused = urlProvider.Received(1).GetUrlAsync(); }
public ClusterMetricsInfo(ResourceManager rm) { // JAXB needs this ResourceScheduler rs = rm.GetResourceScheduler(); QueueMetrics metrics = rs.GetRootQueueMetrics(); ClusterMetrics clusterMetrics = ClusterMetrics.GetMetrics(); this.appsSubmitted = metrics.GetAppsSubmitted(); this.appsCompleted = metrics.GetAppsCompleted(); this.appsPending = metrics.GetAppsPending(); this.appsRunning = metrics.GetAppsRunning(); this.appsFailed = metrics.GetAppsFailed(); this.appsKilled = metrics.GetAppsKilled(); this.reservedMB = metrics.GetReservedMB(); this.availableMB = metrics.GetAvailableMB(); this.allocatedMB = metrics.GetAllocatedMB(); this.reservedVirtualCores = metrics.GetReservedVirtualCores(); this.availableVirtualCores = metrics.GetAvailableVirtualCores(); this.allocatedVirtualCores = metrics.GetAllocatedVirtualCores(); this.containersAllocated = metrics.GetAllocatedContainers(); this.containersPending = metrics.GetPendingContainers(); this.containersReserved = metrics.GetReservedContainers(); this.totalMB = availableMB + allocatedMB; this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores; this.activeNodes = clusterMetrics.GetNumActiveNMs(); this.lostNodes = clusterMetrics.GetNumLostNMs(); this.unhealthyNodes = clusterMetrics.GetUnhealthyNMs(); this.decommissionedNodes = clusterMetrics.GetNumDecommisionedNMs(); this.rebootedNodes = clusterMetrics.GetNumRebootedNMs(); this.totalNodes = activeNodes + lostNodes + decommissionedNodes + rebootedNodes + unhealthyNodes; }
public void PredictClusters() { int n = 1000; int k = 4; var rand = new Random(1); var clusters = new ClusteringData[k]; var data = new ClusteringData[n]; for (int i = 0; i < k; i++) { //pick clusters as points on circle with angle to axis X equal to 360*i/k clusters[i] = new ClusteringData { Points = new float[2] { (float)Math.Cos(Math.PI * i * 2 / k), (float)Math.Sin(Math.PI * i * 2 / k) } }; } // create data points by randomly picking cluster and shifting point slightly away from it. for (int i = 0; i < n; i++) { var index = rand.Next(0, k); var shift = (rand.NextDouble() - 0.5) / 10; data[i] = new ClusteringData { Points = new float[2] { (float)(clusters[index].Points[0] + shift), (float)(clusters[index].Points[1] + shift) } }; } var pipeline = new LearningPipeline(seed: 1, conc: 1); pipeline.Add(CollectionDataSource.Create(data)); pipeline.Add(new KMeansPlusPlusClusterer() { K = k }); var model = pipeline.Train <ClusteringData, ClusteringPrediction>(); //validate that initial points we pick up as centers of cluster during data generation belong to different clusters. var labels = new HashSet <uint>(); for (int i = 0; i < k; i++) { var scores = model.Predict(clusters[i]); Assert.True(!labels.Contains(scores.SelectedClusterId)); labels.Add(scores.SelectedClusterId); } var evaluator = new ClusterEvaluator(); var testData = CollectionDataSource.Create(clusters); ClusterMetrics metrics = evaluator.Evaluate(model, testData); //Label is not specified, so NMI would be equal to NaN Assert.Equal(metrics.Nmi, double.NaN); //Calculate dbi is false by default so Dbi would be 0 Assert.Equal(metrics.Dbi, (double)0.0); Assert.Equal(metrics.AvgMinScore, (double)0.0, 5); }
public MetricsActorSpec() : base(CLUSTERSHARDCONFIG) { cluster = Cluster.Get(Sys); clusterMetrics = ClusterMetrics.Get(Sys); ClusterSharding.Get(Sys).Start(typeName: ShardRegions.OBJECTREGION, entityProps: Props.Create(() => new ObjectActor(null)), settings: ClusterShardingSettings.Create(Sys), messageExtractor: new ObjectRegionMessageExtractor(10)); ClusterSharding.Get(Sys).Start(typeName: ShardRegions.AREAREGION, entityProps: Props.Create(() => new ObjectActor(null)), settings: ClusterShardingSettings.Create(Sys), messageExtractor: new AreaRegionMessageExtractor(10)); }
/// <summary> /// Initializes a new instance of the <see cref="MetricsActor"/> class. /// </summary> /// <param name="interval">The interval.</param> /// <param name="cluster">The cluster.</param> /// <param name="clusterMetrics">The cluster metrics.</param> public MetricsActor(TimeSpan interval, Cluster cluster, ClusterMetrics clusterMetrics) { this.cluster = cluster; this.clusterMetrics = clusterMetrics; this.state = new MetricsState(); this.interval = interval; this.Stash = new UnboundedStashImpl(Context); this.Become(() => this.Start()); }
public void ClusterMetrics_Should_not_collect_publish_and_gossip_metrics_when_disabled() { AwaitClusterUp(Roles.ToArray()); MetricsView.ClusterMetrics.Count.Should().Be(0); ClusterMetrics.Get(Sys).Subscribe(TestActor); ExpectNoMsg(); MetricsView.ClusterMetrics.Count.Should().Be(0); EnterBarrier("after"); }
public ClusterMetricsExtensionSpec(ITestOutputHelper output) : base(ClusterMetricsTestConfig.ClusterConfiguration, output) { var cluster = Cluster.Get(Sys); _extension = ClusterMetrics.Get(Sys); _metricsView = new ClusterMetricsView(cluster.System); _sampleInterval = _extension.Settings.CollectorSampleInterval; }
private void CreateMetricsActor() { Cluster cluster = Cluster.Get(this.system); ClusterMetrics clusterMetrics = ClusterMetrics.Get(this.system); Props props = Props.Create(() => new MetricsActor( TimeSpan.FromSeconds(this.config.Value.Metrics.IntervalSeconds), cluster, clusterMetrics)); IActorRef serverActor = this.system.ActorOf(props, "MetricsActor"); }
public static void InitClusterMetrics() { ClusterMetrics clusterMetrics = ClusterMetrics.GetMetrics(); clusterMetrics.IncrDecommisionedNMs(); clusterMetrics.IncrNumActiveNodes(); clusterMetrics.IncrNumLostNMs(); clusterMetrics.IncrNumRebootedNMs(); clusterMetrics.IncrNumUnhealthyNMs(); }
public ClusterMetricsView(ExtendedActorSystem system) { _system = system; _extension = Metrics.ClusterMetrics.Get(system); _eventBusListener = system.SystemActorOf( Props.Create(() => new EventBusListenerActor(_extension, AppendMetricsChange)) .WithDispatcher(Dispatchers.DefaultDispatcherId) .WithDeploy(Deploy.Local), name: "metrics-event-bus-listener"); }
public void EvaluateModel(IEnumerable <PivotData> testData, PredictionModel <PivotData, ClusteringPrediction> model) { ConsoleWriteHeader("Metrics for Customer Segmentation"); var testDataSource = CollectionDataSource.Create(testData); var evaluator = new ClusterEvaluator(); ClusterMetrics metrics = evaluator.Evaluate(model, testDataSource); Console.WriteLine($"Average mean score: {metrics.AvgMinScore:0.##}"); //Console.WriteLine($"* Davies-Bouldin Index: {metrics.Dbi:#.##}"); //Console.WriteLine($"* Normalized mutual information: {metrics.Nmi:#.##}"); }
/// <summary> /// Retrieves logical properties and measurements of each glyph cluster. /// </summary> /// <remarks> /// If maxClusterCount is not large enough, then E_NOT_SUFFICIENT_BUFFER, which is equivalent to HRESULT_FROM_WIN32(ERROR_INSUFFICIENT_BUFFER), is returned and actualClusterCount is set to the number of clusters needed. /// </remarks> /// <returns>Returns metrics, such as line-break or total advance width, for a glyph cluster. </returns> /// <unmanaged>HRESULT IDWriteTextLayout::GetClusterMetrics([Out, Buffer, Optional] DWRITE_CLUSTER_METRICS* clusterMetrics,[None] int maxClusterCount,[Out] int* actualClusterCount)</unmanaged> public ClusterMetrics[] GetClusterMetrics() { var clusterMetrics = new ClusterMetrics[0]; int clusterCount = 0; int maxClusterCount = 0; GetClusterMetrics(clusterMetrics, clusterCount, out maxClusterCount); if (maxClusterCount > 0) { clusterMetrics = new ClusterMetrics[maxClusterCount]; GetClusterMetrics(clusterMetrics, maxClusterCount, out maxClusterCount); } return(clusterMetrics); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual ClusterMetrics GetClusterMetrics() { try { YarnClusterMetrics metrics = client.GetYarnClusterMetrics(); ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, metrics.GetNumNodeManagers () * 10, metrics.GetNumNodeManagers() * 2, 1, metrics.GetNumNodeManagers(), 0, 0 ); return(oldMetrics); } catch (YarnException e) { throw new IOException(e); } }
public void Evaluate(int SpreadMax) { if (this.FInText.IsConnected == false) { this.metricsCount.SliceCount = 0; this.FCanWrapLineAfter.SliceCount = 0; this.FIsRightToLeft.SliceCount = 0; this.FIsSoftHyphent.SliceCount = 0; this.FIsWhitespace.SliceCount = 0; this.FLength.SliceCount = 0; this.FNewLine.SliceCount = 0; this.FWidth.SliceCount = 0; return; } if (this.FInText.IsChanged) { this.metricsCount.SliceCount = SpreadMax; cm.Clear(); for (int i = 0; i < SpreadMax; i++) { TextLayout tl = this.FInText[i]; ClusterMetrics[] cms = tl.GetClusterMetrics(); this.metricsCount[i] = cms.Length; cm.AddRange(cms); } this.FCanWrapLineAfter.SliceCount = cm.Count; this.FIsRightToLeft.SliceCount = cm.Count; this.FIsSoftHyphent.SliceCount = cm.Count; this.FIsWhitespace.SliceCount = cm.Count; this.FLength.SliceCount = cm.Count; this.FNewLine.SliceCount = cm.Count; this.FWidth.SliceCount = cm.Count; for (int i = 0; i < cm.Count; i++) { ClusterMetrics c = cm[i]; this.FCanWrapLineAfter[i] = c.CanWrapLineAfter; this.FIsRightToLeft[i] = c.IsRightToLeft; this.FIsSoftHyphent[i] = c.IsSoftHyphen; this.FIsWhitespace[i] = c.IsWhitespace; this.FLength[i] = c.Length; this.FWidth[i] = c.Width; } } }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> /// <exception cref="System.Exception"/> public virtual void VerifyClusterMetrics(int submittedApps, int completedApps, int reservedMB, int availableMB, int allocMB, int reservedVirtualCores, int availableVirtualCores , int allocVirtualCores, int totalVirtualCores, int containersAlloc, int totalMB , int totalNodes, int lostNodes, int unhealthyNodes, int decommissionedNodes, int rebootedNodes, int activeNodes) { ResourceScheduler rs = rm.GetResourceScheduler(); QueueMetrics metrics = rs.GetRootQueueMetrics(); ClusterMetrics clusterMetrics = ClusterMetrics.GetMetrics(); long totalMBExpect = metrics.GetAvailableMB() + metrics.GetAllocatedMB(); long totalVirtualCoresExpect = metrics.GetAvailableVirtualCores() + metrics.GetAllocatedVirtualCores (); NUnit.Framework.Assert.AreEqual("appsSubmitted doesn't match", metrics.GetAppsSubmitted (), submittedApps); NUnit.Framework.Assert.AreEqual("appsCompleted doesn't match", metrics.GetAppsCompleted (), completedApps); NUnit.Framework.Assert.AreEqual("reservedMB doesn't match", metrics.GetReservedMB (), reservedMB); NUnit.Framework.Assert.AreEqual("availableMB doesn't match", metrics.GetAvailableMB (), availableMB); NUnit.Framework.Assert.AreEqual("allocatedMB doesn't match", metrics.GetAllocatedMB (), allocMB); NUnit.Framework.Assert.AreEqual("reservedVirtualCores doesn't match", metrics.GetReservedVirtualCores (), reservedVirtualCores); NUnit.Framework.Assert.AreEqual("availableVirtualCores doesn't match", metrics.GetAvailableVirtualCores (), availableVirtualCores); NUnit.Framework.Assert.AreEqual("allocatedVirtualCores doesn't match", totalVirtualCoresExpect , allocVirtualCores); NUnit.Framework.Assert.AreEqual("containersAllocated doesn't match", 0, containersAlloc ); NUnit.Framework.Assert.AreEqual("totalMB doesn't match", totalMBExpect, totalMB); NUnit.Framework.Assert.AreEqual("totalNodes doesn't match", clusterMetrics.GetNumActiveNMs () + clusterMetrics.GetNumLostNMs() + clusterMetrics.GetNumDecommisionedNMs() + clusterMetrics.GetNumRebootedNMs() + clusterMetrics.GetUnhealthyNMs(), totalNodes ); NUnit.Framework.Assert.AreEqual("lostNodes doesn't match", clusterMetrics.GetNumLostNMs (), lostNodes); NUnit.Framework.Assert.AreEqual("unhealthyNodes doesn't match", clusterMetrics.GetUnhealthyNMs (), unhealthyNodes); NUnit.Framework.Assert.AreEqual("decommissionedNodes doesn't match", clusterMetrics .GetNumDecommisionedNMs(), decommissionedNodes); NUnit.Framework.Assert.AreEqual("rebootedNodes doesn't match", clusterMetrics.GetNumRebootedNMs (), rebootedNodes); NUnit.Framework.Assert.AreEqual("activeNodes doesn't match", clusterMetrics.GetNumActiveNMs (), activeNodes); }
public virtual void TestNMExpiry() { string hostname1 = "localhost1"; string hostname2 = "localhost2"; string hostname3 = "localhost3"; Resource capability = BuilderUtils.NewResource(1024, 1); RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest >(); NodeId nodeId1 = NodeId.NewInstance(hostname1, 0); request1.SetNodeId(nodeId1); request1.SetHttpPort(0); request1.SetResource(capability); resourceTrackerService.RegisterNodeManager(request1); RegisterNodeManagerRequest request2 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest >(); NodeId nodeId2 = NodeId.NewInstance(hostname2, 0); request2.SetNodeId(nodeId2); request2.SetHttpPort(0); request2.SetResource(capability); resourceTrackerService.RegisterNodeManager(request2); int waitCount = 0; while (ClusterMetrics.GetMetrics().GetNumLostNMs() != 2 && waitCount++ < 20) { lock (this) { Sharpen.Runtime.Wait(this, 100); } } NUnit.Framework.Assert.AreEqual(2, ClusterMetrics.GetMetrics().GetNumLostNMs()); request3 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest>(); NodeId nodeId3 = NodeId.NewInstance(hostname3, 0); request3.SetNodeId(nodeId3); request3.SetHttpPort(0); request3.SetResource(capability); resourceTrackerService.RegisterNodeManager(request3); /* test to see if hostanme 3 does not expire */ stopT = false; new TestNMExpiry.ThirdNodeHeartBeatThread(this).Start(); NUnit.Framework.Assert.AreEqual(2, ClusterMetrics.GetMetrics().GetNumLostNMs()); stopT = true; }
public AdaptiveLoadBalancingRouterConfig() { Node1 = Role("node-1"); Node2 = Role("node-2"); Node3 = Role("node-3"); CommonConfig = DebugConfig(on: false) .WithFallback(ConfigurationFactory.ParseString(@" # Enable metrics extension in akka-cluster-metrics. akka.extensions=[""Akka.Cluster.Metrics.ClusterMetricsExtensionProvider, Akka.Cluster.Metrics""] akka.cluster.failure-detector.acceptable-heartbeat-pause = 10s # Use rapid metrics collection. akka.cluster.metrics { collector { sample-interval = 1s gossip-interval = 1s moving-average-half-life = 2s } } # Use metrics extension routing. akka.actor.deployment { /router3 = { router = cluster-metrics-adaptive-pool metrics-selector = cpu nr-of-instances = 9 } /router4 = { router = cluster-metrics-adaptive-pool metrics-selector = ""Akka.Cluster.Metrics.Tests.MultiNode.TestCustomMetricsSelector, Akka.Cluster.Metrics.Tests.MultiNode"" nr-of-instances = 10 cluster { enabled = on max-nr-of-instances-per-node = 2 } } } ")) .WithFallback(ClusterMetrics.DefaultConfig()) .WithFallback(MultiNodeClusterSpec.ClusterConfig()); }
private ClusterMetrics GetClusterMetrics() { var result = new ClusterMetrics(); var nodeTag = ServerStore.NodeTag; result.NodeTag = nodeTag; if (string.IsNullOrWhiteSpace(nodeTag) == false) { result.NodeState = ServerStore.CurrentRachisState; } result.CurrentTerm = ServerStore.Engine.CurrentTerm; result.Index = ServerStore.LastRaftCommitIndex; result.Id = ServerStore.Engine.ClusterId; return(result); }
public MetricListener() { _extension = ClusterMetrics.Get(Context.System); var log = Context.GetLogger(); var cluster = Akka.Cluster.Cluster.Get(Context.System); Receive <ClusterMetricsChanged>(clusterMetrics => { clusterMetrics.NodeMetrics .ForEach(nodeMetrics => { foreach (var metric in nodeMetrics.Metrics) { switch (metric.Name) { case StandardMetrics.MemoryUsed: log.Info($"{nodeMetrics.Address}: Used memory: {metric.Value.DoubleValue / 1024 / 1024} MB"); break; case StandardMetrics.CpuTotalUsage: log.Info($"{nodeMetrics.Address}: CPU Total load: {metric.Value}"); break; case StandardMetrics.CpuProcessUsage: log.Info($"{nodeMetrics.Address}: CPU Process load: {metric.Value}"); break; } } }); }); Receive <ClusterEvent.CurrentClusterState>(_ => { // Ignored }); }
public ClusterMetricsEventArgs(ClusterMetrics clusterMetrics) { ClusterMetrics = clusterMetrics; }
/// <summary> /// Retrieves logical properties and measurements of each glyph cluster. /// </summary> /// <remarks> /// If maxClusterCount is not large enough, then E_NOT_SUFFICIENT_BUFFER, which is equivalent to HRESULT_FROM_WIN32(ERROR_INSUFFICIENT_BUFFER), is returned and actualClusterCount is set to the number of clusters needed. /// </remarks> /// <returns>Returns metrics, such as line-break or total advance width, for a glyph cluster. </returns> /// <unmanaged>HRESULT IDWriteTextLayout::GetClusterMetrics([Out, Buffer, Optional] DWRITE_CLUSTER_METRICS* clusterMetrics,[None] int maxClusterCount,[Out] int* actualClusterCount)</unmanaged> public ClusterMetrics[] GetClusterMetrics() { var clusterMetrics = new ClusterMetrics[0]; int clusterCount = 0; int maxClusterCount = 0; GetClusterMetrics(clusterMetrics, clusterCount, out maxClusterCount); if (maxClusterCount > 0) { clusterMetrics = new ClusterMetrics[maxClusterCount]; GetClusterMetrics(clusterMetrics, maxClusterCount, out maxClusterCount); } return clusterMetrics; }
public EventBusListenerActor(ClusterMetrics extension, Action <ClusterMetricsChanged> onMetricsChanged) { _extension = extension; _onMetricsChanged = onMetricsChanged; }