Exemplo n.º 1
0
        public void MetricsGossip_must_merge_an_existing_metric_set_for_a_node_and_update_node_ring()
        {
            var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
                                     _collector.Sample().Metrics);
            var m2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), StandardMetrics.NewTimestamp(),
                                     _collector.Sample().Metrics);
            var m3 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2556), StandardMetrics.NewTimestamp(),
                                     _collector.Sample().Metrics);
            var m2Updated = m2.Copy(metrics: _collector.Sample().Metrics, timestamp: m2.Timestamp + 1000);

            var g1 = MetricsGossip.Empty + m1 + m2;
            var g2 = MetricsGossip.Empty + m3 + m2Updated;

            Assert.True(g1.Nodes.All(x => x.Address == m1.Address || x.Address == m2.Address));

            //should contain nodes 1,3 and the most recent version of 2
            var mergedGossip = g1.Merge(g2);

            XAssert.Equivalent(mergedGossip.Nodes.Select(x => x.Address),
                               new[] { m1.Address, m2.Address, m3.Address });
            mergedGossip.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
            mergedGossip.NodeMetricsFor(m2.Address).Metrics.ShouldBe(m2Updated.Metrics);
            mergedGossip.NodeMetricsFor(m3.Address).Metrics.ShouldBe(m3.Metrics);
            Assert.True(mergedGossip.Nodes.All(x => x.Metrics.Count > 3));
            mergedGossip.NodeMetricsFor(m2.Address).Timestamp.ShouldBe(m2Updated.Timestamp);
        }
Exemplo n.º 2
0
        public void NodeMetric_should_update_3_nodeMetrics_with_ewma_applied()
        {
            const double decay   = ClusterMetricsTestConfig.DefaultDecayFactor;
            const double epsilon = 0.001;

            var sample1 = new NodeMetrics(_node1, 1, ImmutableHashSet.Create(
                                              NodeMetrics.Types.Metric.Create("a", 1, decay).Value,
                                              NodeMetrics.Types.Metric.Create("b", 4, decay).Value
                                              ));
            var sample2 = new NodeMetrics(_node1, 2, ImmutableHashSet.Create(
                                              NodeMetrics.Types.Metric.Create("a", 2, decay).Value,
                                              NodeMetrics.Types.Metric.Create("c", 5, decay).Value
                                              ));
            var sample3 = new NodeMetrics(_node1, 3, ImmutableHashSet.Create(
                                              NodeMetrics.Types.Metric.Create("a", 3, decay).Value,
                                              NodeMetrics.Types.Metric.Create("d", 6, decay).Value
                                              ));

            var updated = sample1.Update(sample2).Update(sample3);

            updated.Metrics.Should().HaveCount(4);
            updated.Timestamp.Should().Be(sample3.Timestamp);

            updated.Metric("a").Value.Value.LongValue.Should().Be(3);
            updated.Metric("b").Value.Value.LongValue.Should().Be(4);
            updated.Metric("c").Value.Value.LongValue.Should().Be(5);
            updated.Metric("d").Value.Value.LongValue.Should().Be(6);

            updated.Metric("a").Value.SmoothValue.Should().BeApproximately(1.512, epsilon);
            updated.Metric("b").Value.SmoothValue.Should().BeApproximately(4.000, epsilon);
            updated.Metric("c").Value.SmoothValue.Should().BeApproximately(5.000, epsilon);
            updated.Metric("d").Value.SmoothValue.Should().BeApproximately(6.000, epsilon);
        }
Exemplo n.º 3
0
 /// <summary>
 /// <see cref="AsyncSequentialDispatcherLocalNode{TInput,TOutput}"/>
 /// </summary>
 /// <param name="progress">Progress of the current bulk</param>
 /// <param name="cts"><see cref="CancellationTokenSource"/></param>
 /// <param name="circuitBreakerOptions"><see cref="CircuitBreakerOptions"/></param>
 /// <param name="clusterOptions"><see cref="ClusterOptions"/></param>
 /// <param name="logger"><see cref="ILogger"/></param>
 public AsyncSequentialDispatcherLocalNode(
     IProgress <double> progress,
     CancellationTokenSource cts,
     CircuitBreakerOptions circuitBreakerOptions,
     ClusterOptions clusterOptions,
     ILogger logger) : base(
         Policy.Handle <Exception>()
         .AdvancedCircuitBreakerAsync(circuitBreakerOptions.CircuitBreakerFailureThreshold,
                                      circuitBreakerOptions.CircuitBreakerSamplingDuration,
                                      circuitBreakerOptions.CircuitBreakerMinimumThroughput,
                                      circuitBreakerOptions.CircuitBreakerDurationOfBreak,
                                      onBreak: (ex, timespan, context) =>
 {
     logger.LogError(
         $"Batch processor breaker: Breaking the circuit for {timespan.TotalMilliseconds}ms due to {ex.Message}.");
 },
                                      onReset: context =>
 {
     logger.LogInformation(
         "Batch processor breaker: Succeeded, closed the circuit.");
 },
                                      onHalfOpen: () =>
 {
     logger.LogWarning(
         "Batch processor breaker: Half-open, next call is a trial.");
 }), clusterOptions, progress, cts, logger)
 {
     _logger         = logger;
     _clusterOptions = clusterOptions;
     NodeMetrics     = new NodeMetrics(Guid.NewGuid());
 }
Exemplo n.º 4
0
        // TODO: This is where you should write your code which will process every new message
        protected override async Task Process(Message message, NodeMetrics nodeMetrics,
                                              CancellationToken cancellationToken)
        {
            if (!_nodes.ContainsKey(nodeMetrics.Id))
            {
                var progressBar = _clusterProgressBar.Spawn(int.MaxValue, $"Node {nodeMetrics.Id} pending process...",
                                                            _nodeProgressBarOptions);
                _nodes.TryAdd(nodeMetrics.Id, progressBar);
            }

            if (_nodes.TryGetValue(nodeMetrics.Id, out var nodeProgressBar))
            {
                // No need to take care of handling exception here, the circuit breaker and retry policies take care of it on a higher level
                if (Random.Value.Next(0, 100) == 50)
                {
                    // Uncomment to see how resiliency is ensured
                    // throw new Exception("I'm a bad exception and I'm trying to break your execution.");
                }

                // Simulate quite long processing time for each message, but could be stressful I/O, networking, ...
                await Task.Delay(125, cancellationToken);

                if (nodeMetrics.CurrentThroughput > 0L && nodeProgressBar.CurrentTick > 0)
                {
                    nodeProgressBar.MaxTicks = (int)nodeMetrics.TotalItemsProcessed;
                }

                nodeProgressBar.Tick(
                    $"Node {nodeMetrics.Id} ({nodeMetrics.CurrentThroughput} messages/s) processed: {message.Body}");
            }

            // Tick when a message has been processed
            _clusterProgressBar.Tick(
                $"New message processed by the cluster ({_clusterFunc().ClusterMetrics.CurrentThroughput} messages/s): {message.Body}");
        }
Exemplo n.º 5
0
        public void MetricGossip_should_get_the_current_NodeMetrics_if_it_exists_in_the_local_nodes()
        {
            var m1 = new NodeMetrics(new Address("akka", "sys", "a", 2554), NewTimestamp, Collector.Sample().Metrics);
            var g1 = MetricsGossip.Empty + m1;

            g1.NodeMetricsFor(m1.Address).Value.Metrics.ShouldBeEquivalentTo(m1.Metrics);
        }
Exemplo n.º 6
0
 /// <summary>
 /// Extract <see cref="Memory"/> data from nodeMetrics, if the nodeMetrics
 /// contains necessary memory metrics, otherwise it returns <see cref="Option{T}.None"/>.
 /// </summary>
 public static Option<Memory> ExtractMemory(NodeMetrics nodeMetrics)
 {
     return Memory.Decompose(nodeMetrics).Select(data =>
     {
         return new Memory(data.Address, data.Timestamp, data.UsedSmoothValue, data.AvailableSmoothValue, data.MaxRecommendedSmoothValue);
     });
 }
Exemplo n.º 7
0
        public void MetricsGossip_must_merge_peer_metrics()
        {
            var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
                                     _collector.Sample().Metrics);
            var m2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), StandardMetrics.NewTimestamp(),
                                     _collector.Sample().Metrics);

            var g1 = MetricsGossip.Empty + m1 + m2;

            g1.Nodes.Count.ShouldBe(2);
            var beforeMergeNodes = g1.Nodes;

            var m2Updated = m2.Copy(metrics: _collector.Sample().Metrics, timestamp: m2.Timestamp + 1000);
            var g2        = g1 + m2Updated; //merge peers

            g2.Nodes.Count.ShouldBe(2);
            g2.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
            g2.NodeMetricsFor(m2.Address).Metrics.ShouldBe(m2.Metrics);
            foreach (var peer in g2.Nodes)
            {
                if (peer.Address == m2.Address)
                {
                    peer.Timestamp.ShouldBe(m2Updated.Timestamp);
                }
            }
        }
Exemplo n.º 8
0
 /// <summary>
 /// Extract <see cref="Cpu"/> data from nodeMetrics, if the nodeMetrics
 /// contains necessary CPU metrics, otherwise it returns <see cref="Option{T}.None"/>.
 /// </summary>
 public static Option<Cpu> ExtractCpu(NodeMetrics nodeMetrics)
 {
     return Cpu.Decompose(nodeMetrics).Select(data =>
     {
         return new Cpu(data.Address, data.Timestamp, data.CpuProcessUsage, data.CpuTotalUsage, data.Processors);
     });
 }
Exemplo n.º 9
0
 public void WriteNodeMetrics(NodeMetrics nodeMetrics)
 {
     //var writeApi = _provider.GetWriteApi();
     //var task = _writeApi.WriteMeasurementAsync(WritePrecision.S, nodeMetrics);
     _writeApi.WriteMeasurement(WritePrecision.S, nodeMetrics);
     //_tasks.Add(task);
     //task.Wait();
 }
Exemplo n.º 10
0
        public void MetricsGossip_must_get_the_current_NodeMetrics_if_it_exists_in_local_nodes()
        {
            var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
                                     _collector.Sample().Metrics);
            var g1 = MetricsGossip.Empty + m1;

            g1.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
        }
Exemplo n.º 11
0
 /// <summary>
 /// Process the result of both resolvers
 /// </summary>
 /// <param name="location"><see cref="Geolocation"/></param>
 /// <param name="header">Header value</param>
 /// <param name="nodeMetrics"><see cref="NodeMetrics"/></param>
 /// <param name="cancellationToken"><see cref="CancellationToken"/></param>
 /// <returns><see cref="Task"/></returns>
 protected override async Task Process(Geolocation location,
                                       string header,
                                       NodeMetrics nodeMetrics,
                                       CancellationToken cancellationToken)
 {
     _logger.LogInformation(
         $"Header {header} associated to location (lat: {location.Lat};lon: {location.Lon})");
     await Task.CompletedTask;
 }
Exemplo n.º 12
0
 /// <summary>
 /// Process each new header
 /// </summary>
 /// <param name="header">Header</param>
 /// <param name="nodeMetrics"><see cref="NodeMetrics"/></param>
 /// <param name="cancellationToken"><see cref="CancellationToken"/></param>
 /// <returns><see cref="Task"/></returns>
 protected override Task <string> Process(string header,
                                          NodeMetrics nodeMetrics,
                                          CancellationToken cancellationToken)
 {
     _logger.LogInformation(
         $"New headers received, inserting to memory cache identified by request identifier from node {nodeMetrics.Id}...");
     _cache.Set(Guid.NewGuid(), header);
     return(Task.FromResult(header));
 }
Exemplo n.º 13
0
        private void LogMemory(NodeMetrics nodeMetrics)
        {
            Option <StandardMetrics.Memory> memory = StandardMetrics.ExtractMemory(nodeMetrics);

            if (memory.HasValue)
            {
                _log.Info("Used memory: {0} Mb", memory.Value.Used / 1024 / 1024);
            }
        }
Exemplo n.º 14
0
 public void NodeMetrics_must_not_merge_2_NodeMetrics_if_master_is_more_recent()
 {
     var sample1 = new NodeMetrics(_node1, 1,
          ImmutableHashSet.Create<Metric>(Metric.Create("a", 10), Metric.Create("b", 20)));
     var sample2 = new NodeMetrics(_node1, 0, ImmutableHashSet.Create<Metric>(Metric.Create("a", 11), Metric.Create("c", 30)));
     var merged = sample1.Merge(sample2); //older and not the same
     merged.Timestamp.ShouldBe(sample1.Timestamp);
     merged.Metrics.ShouldBe(sample1.Metrics);
 }
Exemplo n.º 15
0
        /// <summary>
        /// <see cref="UnarySequentialDispatcherRemoteNode{TInput}"/>
        /// </summary>
        /// <param name="persistentCache">Persistent cache to avoid dropped data on system crash</param>
        /// <param name="progress">Progress of the current bulk</param>
        /// <param name="host"><see cref="Host"/></param>
        /// <param name="cts"><see cref="CancellationTokenSource"/></param>
        /// <param name="circuitBreakerOptions"><see cref="CircuitBreakerOptions"/></param>
        /// <param name="clusterOptions"><see cref="ClusterOptions"/></param>
        /// <param name="logger"><see cref="ILogger"/></param>
        public UnarySequentialDispatcherRemoteNode(
            IAppCache persistentCache,
            IProgress <double> progress,
            Host host,
            CancellationTokenSource cts,
            CircuitBreakerOptions circuitBreakerOptions,
            ClusterOptions clusterOptions,
            ILogger logger) : base(
                Policy.Handle <Exception>()
                .AdvancedCircuitBreakerAsync(circuitBreakerOptions.CircuitBreakerFailureThreshold,
                                             circuitBreakerOptions.CircuitBreakerSamplingDuration,
                                             circuitBreakerOptions.CircuitBreakerMinimumThroughput,
                                             circuitBreakerOptions.CircuitBreakerDurationOfBreak,
                                             onBreak: (ex, timespan, context) =>
        {
            logger.LogError(
                $"Batch processor breaker: Breaking the circuit for {timespan.TotalMilliseconds}ms due to {ex.Message}.");
        },
                                             onReset: context =>
        {
            logger.LogInformation(
                "Batch processor breaker: Succeeded, closed the circuit.");
        },
                                             onHalfOpen: () =>
        {
            logger.LogWarning(
                "Batch processor breaker: Half-open, next call is a trial.");
        }), clusterOptions, progress, cts, logger)
        {
            _logger         = logger;
            _clusterOptions = clusterOptions;

            ISubject <PersistentItem <TInput> > dispatcherSubject = new Subject <PersistentItem <TInput> >();
            _synchronizedDispatcherSubject             = Subject.Synchronize(dispatcherSubject);
            _synchronizedDispatcherSubjectSubscription = _synchronizedDispatcherSubject
                                                         .ObserveOn(new EventLoopScheduler(ts => new Thread(ts)))
                                                         .Select(item =>
            {
                return(Observable.FromAsync(() => persistentCache.AddItemAsync(item.Entity,
                                                                               item.CancellationTokenSource.Token)));
            })
                                                         .Merge()
                                                         .Subscribe();

            var channel = new Channel(host.MachineName, host.Port,
                                      ChannelCredentials.Insecure);
            _remoteContract = MagicOnionClient.Create <IRemoteContract <TInput> >(channel);
            IRemoteNodeSubject nodeReceiver = new NodeReceiver(_logger);
            _remoteNodeHealthSubscription =
                nodeReceiver.RemoteNodeHealthSubject.Subscribe(remoteNodeHealth =>
            {
                NodeMetrics.RemoteNodeHealth = remoteNodeHealth;
            });
            _nodeHub = StreamingHubClient.Connect <INodeHub, INodeReceiver>(channel, (INodeReceiver)nodeReceiver);

            NodeMetrics = new NodeMetrics(Guid.NewGuid());
        }
Exemplo n.º 16
0
        private void LogCpu(NodeMetrics nodeMetrics)
        {
            Option <StandardMetrics.Cpu> cpu = StandardMetrics.ExtractCpu(nodeMetrics);

            if (cpu.HasValue)
            {
                _log.Info("Cpu load: {0}% ({1} processors)", cpu.Value.TotalUsage / 100, cpu.Value.ProcessorsNumber);
            }
        }
Exemplo n.º 17
0
        private void UpdateMemory(NodeMetrics nodeMetrics)
        {
            Option <StandardMetrics.Memory> memory = StandardMetrics.ExtractMemory(nodeMetrics);

            if (memory.HasValue)
            {
                this.state.Memory = memory.Value;
                this.log.Info("METRICS: memory used: {0:0.00} Mb", memory.Value.Used / 1024 / 1024);
            }
        }
Exemplo n.º 18
0
        protected override async Task Process(Message item, NodeMetrics nodeMetrics, CancellationToken cancellationToken)
        {
            item.Body.Add(Helper.FindPrimeNumber(10));
            if (item.Body.Count == item.Target)
            {
                item.SemaphoreSlim.Release();
            }

            await Task.CompletedTask;
        }
Exemplo n.º 19
0
        public void NodeMetrics_must_not_merge_2_NodeMetrics_if_master_is_more_recent()
        {
            var sample1 = new NodeMetrics(_node1, 1,
                                          ImmutableHashSet.Create <Metric>(Metric.Create("a", 10), Metric.Create("b", 20)));
            var sample2 = new NodeMetrics(_node1, 0, ImmutableHashSet.Create <Metric>(Metric.Create("a", 11), Metric.Create("c", 30)));
            var merged  = sample1.Merge(sample2); //older and not the same

            merged.Timestamp.ShouldBe(sample1.Timestamp);
            merged.Metrics.ShouldBe(sample1.Metrics);
        }
Exemplo n.º 20
0
        private void UpdateCpu(NodeMetrics nodeMetrics)
        {
            Option <StandardMetrics.Cpu> cpu = StandardMetrics.ExtractCpu(nodeMetrics);

            if (cpu.HasValue)
            {
                this.state.Cpu = cpu.Value;
                this.log.Info("METRICS: cpu load: {0:0.00}% ({1} processors)", this.state.Cpu.TotalUsage / 100, this.state.Cpu.ProcessorsNumber);
            }
        }
Exemplo n.º 21
0
 public void NodeMetrics_must_merge_2_NodeMetrics_by_most_recent()
 {
     var sample1 = new NodeMetrics(_node1, 1,
         ImmutableHashSet.Create<Metric>(Metric.Create("a", 10), Metric.Create("b", 20)));
     var sample2 = new NodeMetrics(_node1, 2, ImmutableHashSet.Create<Metric>(Metric.Create("a", 11), Metric.Create("c", 30)));
     var merged = sample1.Merge(sample2);
     merged.Timestamp.ShouldBe(sample2.Timestamp);
     merged.Metric("a").Value.ShouldBe(11);
     merged.Metric("b").Value.ShouldBe(20);
     merged.Metric("c").Value.ShouldBe(30);
 }
Exemplo n.º 22
0
        public void NodeMetrics_must_merge_2_NodeMetrics_by_most_recent()
        {
            var sample1 = new NodeMetrics(_node1, 1,
                                          ImmutableHashSet.Create <Metric>(Metric.Create("a", 10), Metric.Create("b", 20)));
            var sample2 = new NodeMetrics(_node1, 2, ImmutableHashSet.Create <Metric>(Metric.Create("a", 11), Metric.Create("c", 30)));
            var merged  = sample1.Merge(sample2);

            merged.Timestamp.ShouldBe(sample2.Timestamp);
            merged.Metric("a").Value.ShouldBe(11);
            merged.Metric("b").Value.ShouldBe(20);
            merged.Metric("c").Value.ShouldBe(30);
        }
Exemplo n.º 23
0
        /// <summary>
        /// Process each new IP address to resolve its geolocation
        /// </summary>
        /// <param name="item"><see cref="KeyValuePair{TKey,TValue}"/></param>
        /// <param name="nodeMetrics"><see cref="NodeMetrics"/></param>
        /// <param name="cancellationToken"><see cref="CancellationToken"/></param>
        /// <returns><see cref="Task"/></returns>
        protected override async Task <Geolocation> Process(IPAddress item,
                                                            NodeMetrics nodeMetrics,
                                                            CancellationToken cancellationToken)
        {
            _logger.LogInformation(
                $"New IP {item} received, trying to resolve geolocation from node {nodeMetrics.Id}...");
            var response = await _httpClient.GetAsync(new Uri($"http://ip-api.com/json/{item}"),
                                                      cancellationToken);

            var geolocation = JsonConvert.DeserializeObject <Geolocation>(await response.Content.ReadAsStringAsync());

            return(geolocation);
        }
Exemplo n.º 24
0
        /// <summary>
        /// Process each new payload
        /// </summary>
        /// <param name="movieDetails"><see cref="MovieDetails"/></param>
        /// <param name="nodeMetrics"><see cref="NodeMetrics"/></param>
        /// <returns><see cref="UnaryResult{TResult}"/></returns>
        public override async UnaryResult <MovieDetails> ProcessItem1Remotely(MovieDetails movieDetails,
                                                                              NodeMetrics nodeMetrics)
        {
            _logger.LogInformation(
                $"Movie title received from node {nodeMetrics.Id}: {movieDetails.Title}.");
            var movie = await _tmdbClient.SearchMovieAsync(movieDetails.Title);

            return(new MovieDetails
            {
                Title = movieDetails.Title,
                Overview = movie.Results.First().Overview
            });
        }
Exemplo n.º 25
0
        /// <inheritdoc />
        public INodeMetrics GetOrCreateNodeMetrics(Host host)
        {
            var value = _nodeMetricsCollection.GetOrAdd(host, h =>
            {
                var nodeBucket = $"{_sessionBucket}.nodes.{MetricsManager.BuildHostAddressMetricPath(host.Address)}";

                var newRegistry = new NodeMetrics(_driverMetricsProvider, _metricsOptions, _metricsEnabled, nodeBucket);
                _nodeMetricsRegistryCollection.Add(host, newRegistry.MetricsRegistry);

                return(newRegistry);
            });

            return(value);
        }
Exemplo n.º 26
0
        public void MetricGossip_should_filter_nodes()
        {
            var m1 = new NodeMetrics(new Address("akka", "sys", "a", 2554), NewTimestamp, Collector.Sample().Metrics);
            var m2 = new NodeMetrics(new Address("akka", "sys", "a", 2555), NewTimestamp, Collector.Sample().Metrics);

            var g1 = MetricsGossip.Empty + m1 + m2;

            g1.Nodes.Should().HaveCount(2);
            var g2 = g1.Filter(ImmutableHashSet.Create(m2.Address));

            g2.Nodes.Should().HaveCount(1);
            g2.Nodes.Should().NotContain(n => n.Address.Equals(m1.Address));
            g2.NodeMetricsFor(m1.Address).Should().Be(Option <NodeMetrics> .None);
            g2.NodeMetricsFor(m2.Address).Value.Metrics.ShouldBeEquivalentTo(m2.Metrics);
        }
Exemplo n.º 27
0
 public MetricValuesSpec() : base(ClusterMetricsTestConfig.DefaultEnabled)
 {
     _node1 = new NodeMetrics(new Address("akka", "sys", "a", 2554), 1, Collector.Sample().Metrics);
     _node2 = new NodeMetrics(new Address("akka", "sys", "a", 2555), 1, Collector.Sample().Metrics);
     _nodes = Enumerable.Range(1, 100).Aggregate(ImmutableList.Create(_node1, _node2), (nodes, _) =>
     {
         return(nodes.Select(n =>
         {
             return new NodeMetrics(n.Address, n.Timestamp, metrics: Collector.Sample().Metrics.SelectMany(latest =>
             {
                 return n.Metrics.Where(latest.SameAs).Select(streaming => streaming + latest);
             }));
         }).ToImmutableList());
     });
 }
Exemplo n.º 28
0
        public void MetricGossip_should_remove_a_node_if_it_is_no_longer_up()
        {
            var m1 = new NodeMetrics(new Address("akka", "sys", "a", 2554), NewTimestamp, Collector.Sample().Metrics);
            var m2 = new NodeMetrics(new Address("akka", "sys", "a", 2555), NewTimestamp, Collector.Sample().Metrics);

            var g1 = MetricsGossip.Empty + m1 + m2;

            g1.Nodes.Should().HaveCount(2);
            var g2 = g1.Remove(m1.Address);

            g2.Nodes.Should().HaveCount(1);
            g2.Nodes.Should().NotContain(n => n.Address.Equals(m1.Address));
            g2.NodeMetricsFor(m1.Address).Should().Be(Option <NodeMetrics> .None);
            g2.NodeMetricsFor(m2.Address).Value.Metrics.ShouldBeEquivalentTo(m2.Metrics);
        }
Exemplo n.º 29
0
                Decompose(NodeMetrics nodeMetrics)
            {
                var used = nodeMetrics.Metric(MemoryUsed);
                var available = nodeMetrics.Metric(MemoryAvailable);
                
                if (!used.HasValue || !available.HasValue)
                    return Option<(Actor.Address, long, double, double, Option<double>)>.None;

                return (
                    nodeMetrics.Address,
                    nodeMetrics.Timestamp,
                    used.Value.SmoothValue,
                    available.Value.SmoothValue,
                    nodeMetrics.Metric(MaxMemoryRecommended).Select(v => v.SmoothValue)
                );
            }
Exemplo n.º 30
0
        public void NodeMetrics_should_not_merge_2_nodeMetrics_if_master_is_more_resent()
        {
            var sample1 = new NodeMetrics(_node1, 1, ImmutableHashSet.Create(
                                              NodeMetrics.Types.Metric.Create("a", 10, Option <double> .None).Value,
                                              NodeMetrics.Types.Metric.Create("b", 20, Option <double> .None).Value
                                              ));
            var sample2 = new NodeMetrics(_node1, 0, ImmutableHashSet.Create(
                                              NodeMetrics.Types.Metric.Create("a", 11, Option <double> .None).Value,
                                              NodeMetrics.Types.Metric.Create("c", 30, Option <double> .None).Value
                                              ));

            var merged = sample1.Merge(sample2);

            merged.Timestamp.Should().Be(sample1.Timestamp);
            merged.Metrics.ShouldBeEquivalentTo(sample1.Metrics);
        }
Exemplo n.º 31
0
        /// <summary>
        /// <see cref="UnarySequentialDispatcherLocalNode{TInput}"/>
        /// </summary>
        /// <param name="persistentCache">Persistent cache to avoid dropped data on system crash</param>
        /// <param name="process">The <see cref="Task"/> to be applied to an item</param>
        /// <param name="progress">Progress of the current bulk</param>
        /// <param name="cts"><see cref="CancellationTokenSource"/></param>
        /// <param name="circuitBreakerOptions"><see cref="CircuitBreakerOptions"/></param>
        /// <param name="clusterOptions"><see cref="ClusterOptions"/></param>
        /// <param name="logger"><see cref="ILogger"/></param>
        public UnarySequentialDispatcherLocalNode(
            IAppCache persistentCache,
            Func <TInput, NodeMetrics, CancellationToken, Task> process,
            IProgress <double> progress,
            CancellationTokenSource cts,
            CircuitBreakerOptions circuitBreakerOptions,
            ClusterOptions clusterOptions,
            ILogger logger) : base(
                Policy.Handle <Exception>()
                .AdvancedCircuitBreakerAsync(circuitBreakerOptions.CircuitBreakerFailureThreshold,
                                             circuitBreakerOptions.CircuitBreakerSamplingDuration,
                                             circuitBreakerOptions.CircuitBreakerMinimumThroughput,
                                             circuitBreakerOptions.CircuitBreakerDurationOfBreak,
                                             onBreak: (ex, timespan, context) =>
        {
            logger.LogError(
                $"Batch processor breaker: Breaking the circuit for {timespan.TotalMilliseconds}ms due to {ex.Message}.");
        },
                                             onReset: context =>
        {
            logger.LogInformation(
                "Batch processor breaker: Succeeded, closed the circuit.");
        },
                                             onHalfOpen: () =>
        {
            logger.LogWarning(
                "Batch processor breaker: Half-open, next call is a trial.");
        }), clusterOptions, progress, cts, logger)
        {
            _logger         = logger;
            _process        = process;
            _clusterOptions = clusterOptions;

            ISubject <PersistentItem <TInput> > dispatcherSubject = new Subject <PersistentItem <TInput> >();
            _synchronizedDispatcherSubject             = Subject.Synchronize(dispatcherSubject);
            _synchronizedDispatcherSubjectSubscription = _synchronizedDispatcherSubject
                                                         .ObserveOn(new EventLoopScheduler(ts => new Thread(ts)))
                                                         .Select(item =>
            {
                return(Observable.FromAsync(() => persistentCache.AddItemAsync(item.Entity,
                                                                               item.CancellationTokenSource.Token)));
            })
                                                         .Merge()
                                                         .Subscribe();

            NodeMetrics = new NodeMetrics(Guid.NewGuid());
        }
Exemplo n.º 32
0
        public void MetricGossip_should_merge_peer_metrics()
        {
            var m1 = new NodeMetrics(new Address("akka", "sys", "a", 2554), NewTimestamp, Collector.Sample().Metrics);
            var m2 = new NodeMetrics(new Address("akka", "sys", "a", 2555), NewTimestamp, Collector.Sample().Metrics);

            var g1 = MetricsGossip.Empty + m1 + m2;

            g1.Nodes.Should().HaveCount(2);

            var m2Updated = new NodeMetrics(m2.Address, m2.Timestamp + 1000, NewSample(m2.Metrics));
            var g2        = g1 + m2Updated; // merge peers

            g2.Nodes.Should().HaveCount(2);
            g2.NodeMetricsFor(m1.Address).Value.Metrics.Should().BeEquivalentTo(m1.Metrics);
            g2.NodeMetricsFor(m2.Address).Value.Metrics.Should().BeEquivalentTo(m2Updated.Metrics);
            g2.Nodes.Where(p => p.Address.Equals(m2.Address)).ForEach(p => p.Timestamp.Should().Be(m2Updated.Timestamp));
        }
Exemplo n.º 33
0
        public void MetricsGossip_must_add_new_NodeMetrics()
        {
            var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
                _collector.Sample().Metrics);
            var m2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), StandardMetrics.NewTimestamp(),
                _collector.Sample().Metrics);

            Assert.True(m1.Metrics.Count > 3);
            Assert.True(m2.Metrics.Count > 3);

            var g1 = MetricsGossip.Empty + m1;
            g1.Nodes.Count.ShouldBe(1);
            g1.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);

            var g2 = g1 + m2;
            g2.Nodes.Count.ShouldBe(2);
            g2.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
            g2.NodeMetricsFor(m2.Address).Metrics.ShouldBe(m2.Metrics);
        }
Exemplo n.º 34
0
        public void MetricsGossip_must_merge_peer_metrics()
        {
            var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
                _collector.Sample().Metrics);
            var m2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), StandardMetrics.NewTimestamp(),
                _collector.Sample().Metrics);

            var g1 = MetricsGossip.Empty + m1 + m2;
            g1.Nodes.Count.ShouldBe(2);
            var beforeMergeNodes = g1.Nodes;

            var m2Updated = m2.Copy(metrics: _collector.Sample().Metrics, timestamp: m2.Timestamp + 1000);
            var g2 = g1 + m2Updated; //merge peers
            g2.Nodes.Count.ShouldBe(2);
            g2.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
            g2.NodeMetricsFor(m2.Address).Metrics.ShouldBe(m2.Metrics);
            foreach (var peer in g2.Nodes)
            {
                if(peer.Address == m2.Address)
                    peer.Timestamp.ShouldBe(m2Updated.Timestamp);
            }
        }
Exemplo n.º 35
0
        public void MetricsGossip_must_merge_an_existing_metric_set_for_a_node_and_update_node_ring()
        {
            var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
                _collector.Sample().Metrics);
            var m2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), StandardMetrics.NewTimestamp(),
                _collector.Sample().Metrics);
            var m3 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2556), StandardMetrics.NewTimestamp(),
                _collector.Sample().Metrics);
            var m2Updated = m2.Copy(metrics: _collector.Sample().Metrics, timestamp: m2.Timestamp + 1000);

            var g1 = MetricsGossip.Empty + m1 + m2;
            var g2 = MetricsGossip.Empty + m3 + m2Updated;
            Assert.True(g1.Nodes.All(x => x.Address == m1.Address || x.Address == m2.Address));

            //should contain nodes 1,3 and the most recent version of 2
            var mergedGossip = g1.Merge(g2);
            XAssert.Equivalent(mergedGossip.Nodes.Select(x => x.Address),
                new[] {m1.Address, m2.Address, m3.Address});
            mergedGossip.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
            mergedGossip.NodeMetricsFor(m2.Address).Metrics.ShouldBe(m2Updated.Metrics);
            mergedGossip.NodeMetricsFor(m3.Address).Metrics.ShouldBe(m3.Metrics);
            Assert.True(mergedGossip.Nodes.All(x => x.Metrics.Count > 3));
            mergedGossip.NodeMetricsFor(m2.Address).Timestamp.ShouldBe(m2Updated.Timestamp);
        }
Exemplo n.º 36
0
 public void MetricsGossip_must_get_the_current_NodeMetrics_if_it_exists_in_local_nodes()
 {
     var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
        _collector.Sample().Metrics);
     var g1 = MetricsGossip.Empty + m1;
     g1.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
 }
Exemplo n.º 37
0
        public void MetricsGossip_must_filter_nodes()
        {
            var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
                _collector.Sample().Metrics);
            var m2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), StandardMetrics.NewTimestamp(),
                _collector.Sample().Metrics);

            var g1 = MetricsGossip.Empty + m1 + m2;
            g1.Nodes.Count.ShouldBe(2);
            var g2 = g1.Filter(ImmutableHashSet.Create<Address>(new[] {m2.Address}));
            g2.Nodes.Count.ShouldBe(1);
            g2.Nodes.Any(x => x.Address == m1.Address).ShouldBeFalse();
            g2.NodeMetricsFor(m1.Address).ShouldBe(null);
            g2.NodeMetricsFor(m2.Address).Metrics.ShouldBe(m2.Metrics);
        }
Exemplo n.º 38
0
        public void MetricsGossip_must_remove_a_node_if_it_is_no_longer_up()
        {
            var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
                _collector.Sample().Metrics);
            var m2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), StandardMetrics.NewTimestamp(),
                _collector.Sample().Metrics);

            var g1 = MetricsGossip.Empty + m1 + m2;
            g1.Nodes.Count.ShouldBe(2);
            var g2 = g1.Remove(m1.Address);
            g2.Nodes.Count.ShouldBe(1);
            g2.Nodes.Any(x => x.Address == m1.Address).ShouldBeFalse();
            g2.NodeMetricsFor(m1.Address).ShouldBe(null);
            g2.NodeMetricsFor(m2.Address).Metrics.ShouldBe(m2.Metrics);
        }
Exemplo n.º 39
0
 public MetricValuesSpec()
 {
     _collector = CreateMetricsCollector();
     _node1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), 1, _collector.Sample().Metrics);
     _node2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), 1, _collector.Sample().Metrics);
 }