public void MetricsGossip_must_merge_an_existing_metric_set_for_a_node_and_update_node_ring()
{
var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var m2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var m3 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2556), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var m2Updated = m2.Copy(metrics: _collector.Sample().Metrics, timestamp: m2.Timestamp + 1000);
var g1 = MetricsGossip.Empty + m1 + m2;
var g2 = MetricsGossip.Empty + m3 + m2Updated;
Assert.True(g1.Nodes.All(x => x.Address == m1.Address || x.Address == m2.Address));
//should contain nodes 1,3 and the most recent version of 2
var mergedGossip = g1.Merge(g2);
XAssert.Equivalent(mergedGossip.Nodes.Select(x => x.Address),
new[] { m1.Address, m2.Address, m3.Address });
mergedGossip.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
mergedGossip.NodeMetricsFor(m2.Address).Metrics.ShouldBe(m2Updated.Metrics);
mergedGossip.NodeMetricsFor(m3.Address).Metrics.ShouldBe(m3.Metrics);
Assert.True(mergedGossip.Nodes.All(x => x.Metrics.Count > 3));
mergedGossip.NodeMetricsFor(m2.Address).Timestamp.ShouldBe(m2Updated.Timestamp);
}
public void NodeMetric_should_update_3_nodeMetrics_with_ewma_applied()
{
const double decay = ClusterMetricsTestConfig.DefaultDecayFactor;
const double epsilon = 0.001;
var sample1 = new NodeMetrics(_node1, 1, ImmutableHashSet.Create(
NodeMetrics.Types.Metric.Create("a", 1, decay).Value,
NodeMetrics.Types.Metric.Create("b", 4, decay).Value
));
var sample2 = new NodeMetrics(_node1, 2, ImmutableHashSet.Create(
NodeMetrics.Types.Metric.Create("a", 2, decay).Value,
NodeMetrics.Types.Metric.Create("c", 5, decay).Value
));
var sample3 = new NodeMetrics(_node1, 3, ImmutableHashSet.Create(
NodeMetrics.Types.Metric.Create("a", 3, decay).Value,
NodeMetrics.Types.Metric.Create("d", 6, decay).Value
));
var updated = sample1.Update(sample2).Update(sample3);
updated.Metrics.Should().HaveCount(4);
updated.Timestamp.Should().Be(sample3.Timestamp);
updated.Metric("a").Value.Value.LongValue.Should().Be(3);
updated.Metric("b").Value.Value.LongValue.Should().Be(4);
updated.Metric("c").Value.Value.LongValue.Should().Be(5);
updated.Metric("d").Value.Value.LongValue.Should().Be(6);
updated.Metric("a").Value.SmoothValue.Should().BeApproximately(1.512, epsilon);
updated.Metric("b").Value.SmoothValue.Should().BeApproximately(4.000, epsilon);
updated.Metric("c").Value.SmoothValue.Should().BeApproximately(5.000, epsilon);
updated.Metric("d").Value.SmoothValue.Should().BeApproximately(6.000, epsilon);
}
/// <summary>
/// <see cref="AsyncSequentialDispatcherLocalNode{TInput,TOutput}"/>
/// </summary>
/// <param name="progress">Progress of the current bulk</param>
/// <param name="cts"><see cref="CancellationTokenSource"/></param>
/// <param name="circuitBreakerOptions"><see cref="CircuitBreakerOptions"/></param>
/// <param name="clusterOptions"><see cref="ClusterOptions"/></param>
/// <param name="logger"><see cref="ILogger"/></param>
public AsyncSequentialDispatcherLocalNode(
IProgress <double> progress,
CancellationTokenSource cts,
CircuitBreakerOptions circuitBreakerOptions,
ClusterOptions clusterOptions,
ILogger logger) : base(
Policy.Handle <Exception>()
.AdvancedCircuitBreakerAsync(circuitBreakerOptions.CircuitBreakerFailureThreshold,
circuitBreakerOptions.CircuitBreakerSamplingDuration,
circuitBreakerOptions.CircuitBreakerMinimumThroughput,
circuitBreakerOptions.CircuitBreakerDurationOfBreak,
onBreak: (ex, timespan, context) =>
{
logger.LogError(
$"Batch processor breaker: Breaking the circuit for {timespan.TotalMilliseconds}ms due to {ex.Message}.");
},
onReset: context =>
{
logger.LogInformation(
"Batch processor breaker: Succeeded, closed the circuit.");
},
onHalfOpen: () =>
{
logger.LogWarning(
"Batch processor breaker: Half-open, next call is a trial.");
}), clusterOptions, progress, cts, logger)
{
_logger = logger;
_clusterOptions = clusterOptions;
NodeMetrics = new NodeMetrics(Guid.NewGuid());
}
// TODO: This is where you should write your code which will process every new message
protected override async Task Process(Message message, NodeMetrics nodeMetrics,
CancellationToken cancellationToken)
{
if (!_nodes.ContainsKey(nodeMetrics.Id))
{
var progressBar = _clusterProgressBar.Spawn(int.MaxValue, $"Node {nodeMetrics.Id} pending process...",
_nodeProgressBarOptions);
_nodes.TryAdd(nodeMetrics.Id, progressBar);
}
if (_nodes.TryGetValue(nodeMetrics.Id, out var nodeProgressBar))
{
// No need to take care of handling exception here, the circuit breaker and retry policies take care of it on a higher level
if (Random.Value.Next(0, 100) == 50)
{
// Uncomment to see how resiliency is ensured
// throw new Exception("I'm a bad exception and I'm trying to break your execution.");
}
// Simulate quite long processing time for each message, but could be stressful I/O, networking, ...
await Task.Delay(125, cancellationToken);
if (nodeMetrics.CurrentThroughput > 0L && nodeProgressBar.CurrentTick > 0)
{
nodeProgressBar.MaxTicks = (int)nodeMetrics.TotalItemsProcessed;
}
nodeProgressBar.Tick(
$"Node {nodeMetrics.Id} ({nodeMetrics.CurrentThroughput} messages/s) processed: {message.Body}");
}
// Tick when a message has been processed
_clusterProgressBar.Tick(
$"New message processed by the cluster ({_clusterFunc().ClusterMetrics.CurrentThroughput} messages/s): {message.Body}");
}
public void MetricGossip_should_get_the_current_NodeMetrics_if_it_exists_in_the_local_nodes()
{
var m1 = new NodeMetrics(new Address("akka", "sys", "a", 2554), NewTimestamp, Collector.Sample().Metrics);
var g1 = MetricsGossip.Empty + m1;
g1.NodeMetricsFor(m1.Address).Value.Metrics.ShouldBeEquivalentTo(m1.Metrics);
}
/// <summary>
/// Extract <see cref="Memory"/> data from nodeMetrics, if the nodeMetrics
/// contains necessary memory metrics, otherwise it returns <see cref="Option{T}.None"/>.
/// </summary>
public static Option<Memory> ExtractMemory(NodeMetrics nodeMetrics)
{
return Memory.Decompose(nodeMetrics).Select(data =>
{
return new Memory(data.Address, data.Timestamp, data.UsedSmoothValue, data.AvailableSmoothValue, data.MaxRecommendedSmoothValue);
});
}
public void MetricsGossip_must_merge_peer_metrics()
{
var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var m2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var g1 = MetricsGossip.Empty + m1 + m2;
g1.Nodes.Count.ShouldBe(2);
var beforeMergeNodes = g1.Nodes;
var m2Updated = m2.Copy(metrics: _collector.Sample().Metrics, timestamp: m2.Timestamp + 1000);
var g2 = g1 + m2Updated; //merge peers
g2.Nodes.Count.ShouldBe(2);
g2.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
g2.NodeMetricsFor(m2.Address).Metrics.ShouldBe(m2.Metrics);
foreach (var peer in g2.Nodes)
{
if (peer.Address == m2.Address)
{
peer.Timestamp.ShouldBe(m2Updated.Timestamp);
}
}
}
/// <summary>
/// Extract <see cref="Cpu"/> data from nodeMetrics, if the nodeMetrics
/// contains necessary CPU metrics, otherwise it returns <see cref="Option{T}.None"/>.
/// </summary>
public static Option<Cpu> ExtractCpu(NodeMetrics nodeMetrics)
{
return Cpu.Decompose(nodeMetrics).Select(data =>
{
return new Cpu(data.Address, data.Timestamp, data.CpuProcessUsage, data.CpuTotalUsage, data.Processors);
});
}
public void WriteNodeMetrics(NodeMetrics nodeMetrics)
{
//var writeApi = _provider.GetWriteApi();
//var task = _writeApi.WriteMeasurementAsync(WritePrecision.S, nodeMetrics);
_writeApi.WriteMeasurement(WritePrecision.S, nodeMetrics);
//_tasks.Add(task);
//task.Wait();
}
public void MetricsGossip_must_get_the_current_NodeMetrics_if_it_exists_in_local_nodes()
{
var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var g1 = MetricsGossip.Empty + m1;
g1.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
}
/// <summary>
/// Process the result of both resolvers
/// </summary>
/// <param name="location"><see cref="Geolocation"/></param>
/// <param name="header">Header value</param>
/// <param name="nodeMetrics"><see cref="NodeMetrics"/></param>
/// <param name="cancellationToken"><see cref="CancellationToken"/></param>
/// <returns><see cref="Task"/></returns>
protected override async Task Process(Geolocation location,
string header,
NodeMetrics nodeMetrics,
CancellationToken cancellationToken)
{
_logger.LogInformation(
$"Header {header} associated to location (lat: {location.Lat};lon: {location.Lon})");
await Task.CompletedTask;
}
/// <summary>
/// Process each new header
/// </summary>
/// <param name="header">Header</param>
/// <param name="nodeMetrics"><see cref="NodeMetrics"/></param>
/// <param name="cancellationToken"><see cref="CancellationToken"/></param>
/// <returns><see cref="Task"/></returns>
protected override Task <string> Process(string header,
NodeMetrics nodeMetrics,
CancellationToken cancellationToken)
{
_logger.LogInformation(
$"New headers received, inserting to memory cache identified by request identifier from node {nodeMetrics.Id}...");
_cache.Set(Guid.NewGuid(), header);
return(Task.FromResult(header));
}
private void LogMemory(NodeMetrics nodeMetrics)
{
Option <StandardMetrics.Memory> memory = StandardMetrics.ExtractMemory(nodeMetrics);
if (memory.HasValue)
{
_log.Info("Used memory: {0} Mb", memory.Value.Used / 1024 / 1024);
}
}
public void NodeMetrics_must_not_merge_2_NodeMetrics_if_master_is_more_recent()
{
var sample1 = new NodeMetrics(_node1, 1,
ImmutableHashSet.Create<Metric>(Metric.Create("a", 10), Metric.Create("b", 20)));
var sample2 = new NodeMetrics(_node1, 0, ImmutableHashSet.Create<Metric>(Metric.Create("a", 11), Metric.Create("c", 30)));
var merged = sample1.Merge(sample2); //older and not the same
merged.Timestamp.ShouldBe(sample1.Timestamp);
merged.Metrics.ShouldBe(sample1.Metrics);
}
/// <summary>
/// <see cref="UnarySequentialDispatcherRemoteNode{TInput}"/>
/// </summary>
/// <param name="persistentCache">Persistent cache to avoid dropped data on system crash</param>
/// <param name="progress">Progress of the current bulk</param>
/// <param name="host"><see cref="Host"/></param>
/// <param name="cts"><see cref="CancellationTokenSource"/></param>
/// <param name="circuitBreakerOptions"><see cref="CircuitBreakerOptions"/></param>
/// <param name="clusterOptions"><see cref="ClusterOptions"/></param>
/// <param name="logger"><see cref="ILogger"/></param>
public UnarySequentialDispatcherRemoteNode(
IAppCache persistentCache,
IProgress <double> progress,
Host host,
CancellationTokenSource cts,
CircuitBreakerOptions circuitBreakerOptions,
ClusterOptions clusterOptions,
ILogger logger) : base(
Policy.Handle <Exception>()
.AdvancedCircuitBreakerAsync(circuitBreakerOptions.CircuitBreakerFailureThreshold,
circuitBreakerOptions.CircuitBreakerSamplingDuration,
circuitBreakerOptions.CircuitBreakerMinimumThroughput,
circuitBreakerOptions.CircuitBreakerDurationOfBreak,
onBreak: (ex, timespan, context) =>
{
logger.LogError(
$"Batch processor breaker: Breaking the circuit for {timespan.TotalMilliseconds}ms due to {ex.Message}.");
},
onReset: context =>
{
logger.LogInformation(
"Batch processor breaker: Succeeded, closed the circuit.");
},
onHalfOpen: () =>
{
logger.LogWarning(
"Batch processor breaker: Half-open, next call is a trial.");
}), clusterOptions, progress, cts, logger)
{
_logger = logger;
_clusterOptions = clusterOptions;
ISubject <PersistentItem <TInput> > dispatcherSubject = new Subject <PersistentItem <TInput> >();
_synchronizedDispatcherSubject = Subject.Synchronize(dispatcherSubject);
_synchronizedDispatcherSubjectSubscription = _synchronizedDispatcherSubject
.ObserveOn(new EventLoopScheduler(ts => new Thread(ts)))
.Select(item =>
{
return(Observable.FromAsync(() => persistentCache.AddItemAsync(item.Entity,
item.CancellationTokenSource.Token)));
})
.Merge()
.Subscribe();
var channel = new Channel(host.MachineName, host.Port,
ChannelCredentials.Insecure);
_remoteContract = MagicOnionClient.Create <IRemoteContract <TInput> >(channel);
IRemoteNodeSubject nodeReceiver = new NodeReceiver(_logger);
_remoteNodeHealthSubscription =
nodeReceiver.RemoteNodeHealthSubject.Subscribe(remoteNodeHealth =>
{
NodeMetrics.RemoteNodeHealth = remoteNodeHealth;
});
_nodeHub = StreamingHubClient.Connect <INodeHub, INodeReceiver>(channel, (INodeReceiver)nodeReceiver);
NodeMetrics = new NodeMetrics(Guid.NewGuid());
}
private void LogCpu(NodeMetrics nodeMetrics)
{
Option <StandardMetrics.Cpu> cpu = StandardMetrics.ExtractCpu(nodeMetrics);
if (cpu.HasValue)
{
_log.Info("Cpu load: {0}% ({1} processors)", cpu.Value.TotalUsage / 100, cpu.Value.ProcessorsNumber);
}
}
private void UpdateMemory(NodeMetrics nodeMetrics)
{
Option <StandardMetrics.Memory> memory = StandardMetrics.ExtractMemory(nodeMetrics);
if (memory.HasValue)
{
this.state.Memory = memory.Value;
this.log.Info("METRICS: memory used: {0:0.00} Mb", memory.Value.Used / 1024 / 1024);
}
}
protected override async Task Process(Message item, NodeMetrics nodeMetrics, CancellationToken cancellationToken)
{
item.Body.Add(Helper.FindPrimeNumber(10));
if (item.Body.Count == item.Target)
{
item.SemaphoreSlim.Release();
}
await Task.CompletedTask;
}
public void NodeMetrics_must_not_merge_2_NodeMetrics_if_master_is_more_recent()
{
var sample1 = new NodeMetrics(_node1, 1,
ImmutableHashSet.Create <Metric>(Metric.Create("a", 10), Metric.Create("b", 20)));
var sample2 = new NodeMetrics(_node1, 0, ImmutableHashSet.Create <Metric>(Metric.Create("a", 11), Metric.Create("c", 30)));
var merged = sample1.Merge(sample2); //older and not the same
merged.Timestamp.ShouldBe(sample1.Timestamp);
merged.Metrics.ShouldBe(sample1.Metrics);
}
private void UpdateCpu(NodeMetrics nodeMetrics)
{
Option <StandardMetrics.Cpu> cpu = StandardMetrics.ExtractCpu(nodeMetrics);
if (cpu.HasValue)
{
this.state.Cpu = cpu.Value;
this.log.Info("METRICS: cpu load: {0:0.00}% ({1} processors)", this.state.Cpu.TotalUsage / 100, this.state.Cpu.ProcessorsNumber);
}
}
public void NodeMetrics_must_merge_2_NodeMetrics_by_most_recent()
{
var sample1 = new NodeMetrics(_node1, 1,
ImmutableHashSet.Create<Metric>(Metric.Create("a", 10), Metric.Create("b", 20)));
var sample2 = new NodeMetrics(_node1, 2, ImmutableHashSet.Create<Metric>(Metric.Create("a", 11), Metric.Create("c", 30)));
var merged = sample1.Merge(sample2);
merged.Timestamp.ShouldBe(sample2.Timestamp);
merged.Metric("a").Value.ShouldBe(11);
merged.Metric("b").Value.ShouldBe(20);
merged.Metric("c").Value.ShouldBe(30);
}
public void NodeMetrics_must_merge_2_NodeMetrics_by_most_recent()
{
var sample1 = new NodeMetrics(_node1, 1,
ImmutableHashSet.Create <Metric>(Metric.Create("a", 10), Metric.Create("b", 20)));
var sample2 = new NodeMetrics(_node1, 2, ImmutableHashSet.Create <Metric>(Metric.Create("a", 11), Metric.Create("c", 30)));
var merged = sample1.Merge(sample2);
merged.Timestamp.ShouldBe(sample2.Timestamp);
merged.Metric("a").Value.ShouldBe(11);
merged.Metric("b").Value.ShouldBe(20);
merged.Metric("c").Value.ShouldBe(30);
}
/// <summary>
/// Process each new IP address to resolve its geolocation
/// </summary>
/// <param name="item"><see cref="KeyValuePair{TKey,TValue}"/></param>
/// <param name="nodeMetrics"><see cref="NodeMetrics"/></param>
/// <param name="cancellationToken"><see cref="CancellationToken"/></param>
/// <returns><see cref="Task"/></returns>
protected override async Task <Geolocation> Process(IPAddress item,
NodeMetrics nodeMetrics,
CancellationToken cancellationToken)
{
_logger.LogInformation(
$"New IP {item} received, trying to resolve geolocation from node {nodeMetrics.Id}...");
var response = await _httpClient.GetAsync(new Uri($"http://ip-api.com/json/{item}"),
cancellationToken);
var geolocation = JsonConvert.DeserializeObject <Geolocation>(await response.Content.ReadAsStringAsync());
return(geolocation);
}
/// <summary>
/// Process each new payload
/// </summary>
/// <param name="movieDetails"><see cref="MovieDetails"/></param>
/// <param name="nodeMetrics"><see cref="NodeMetrics"/></param>
/// <returns><see cref="UnaryResult{TResult}"/></returns>
public override async UnaryResult <MovieDetails> ProcessItem1Remotely(MovieDetails movieDetails,
NodeMetrics nodeMetrics)
{
_logger.LogInformation(
$"Movie title received from node {nodeMetrics.Id}: {movieDetails.Title}.");
var movie = await _tmdbClient.SearchMovieAsync(movieDetails.Title);
return(new MovieDetails
{
Title = movieDetails.Title,
Overview = movie.Results.First().Overview
});
}
/// <inheritdoc />
public INodeMetrics GetOrCreateNodeMetrics(Host host)
{
var value = _nodeMetricsCollection.GetOrAdd(host, h =>
{
var nodeBucket = $"{_sessionBucket}.nodes.{MetricsManager.BuildHostAddressMetricPath(host.Address)}";
var newRegistry = new NodeMetrics(_driverMetricsProvider, _metricsOptions, _metricsEnabled, nodeBucket);
_nodeMetricsRegistryCollection.Add(host, newRegistry.MetricsRegistry);
return(newRegistry);
});
return(value);
}
public void MetricGossip_should_filter_nodes()
{
var m1 = new NodeMetrics(new Address("akka", "sys", "a", 2554), NewTimestamp, Collector.Sample().Metrics);
var m2 = new NodeMetrics(new Address("akka", "sys", "a", 2555), NewTimestamp, Collector.Sample().Metrics);
var g1 = MetricsGossip.Empty + m1 + m2;
g1.Nodes.Should().HaveCount(2);
var g2 = g1.Filter(ImmutableHashSet.Create(m2.Address));
g2.Nodes.Should().HaveCount(1);
g2.Nodes.Should().NotContain(n => n.Address.Equals(m1.Address));
g2.NodeMetricsFor(m1.Address).Should().Be(Option <NodeMetrics> .None);
g2.NodeMetricsFor(m2.Address).Value.Metrics.ShouldBeEquivalentTo(m2.Metrics);
}
public MetricValuesSpec() : base(ClusterMetricsTestConfig.DefaultEnabled)
{
_node1 = new NodeMetrics(new Address("akka", "sys", "a", 2554), 1, Collector.Sample().Metrics);
_node2 = new NodeMetrics(new Address("akka", "sys", "a", 2555), 1, Collector.Sample().Metrics);
_nodes = Enumerable.Range(1, 100).Aggregate(ImmutableList.Create(_node1, _node2), (nodes, _) =>
{
return(nodes.Select(n =>
{
return new NodeMetrics(n.Address, n.Timestamp, metrics: Collector.Sample().Metrics.SelectMany(latest =>
{
return n.Metrics.Where(latest.SameAs).Select(streaming => streaming + latest);
}));
}).ToImmutableList());
});
}
public void MetricGossip_should_remove_a_node_if_it_is_no_longer_up()
{
var m1 = new NodeMetrics(new Address("akka", "sys", "a", 2554), NewTimestamp, Collector.Sample().Metrics);
var m2 = new NodeMetrics(new Address("akka", "sys", "a", 2555), NewTimestamp, Collector.Sample().Metrics);
var g1 = MetricsGossip.Empty + m1 + m2;
g1.Nodes.Should().HaveCount(2);
var g2 = g1.Remove(m1.Address);
g2.Nodes.Should().HaveCount(1);
g2.Nodes.Should().NotContain(n => n.Address.Equals(m1.Address));
g2.NodeMetricsFor(m1.Address).Should().Be(Option <NodeMetrics> .None);
g2.NodeMetricsFor(m2.Address).Value.Metrics.ShouldBeEquivalentTo(m2.Metrics);
}
Decompose(NodeMetrics nodeMetrics)
{
var used = nodeMetrics.Metric(MemoryUsed);
var available = nodeMetrics.Metric(MemoryAvailable);
if (!used.HasValue || !available.HasValue)
return Option<(Actor.Address, long, double, double, Option<double>)>.None;
return (
nodeMetrics.Address,
nodeMetrics.Timestamp,
used.Value.SmoothValue,
available.Value.SmoothValue,
nodeMetrics.Metric(MaxMemoryRecommended).Select(v => v.SmoothValue)
);
}
public void NodeMetrics_should_not_merge_2_nodeMetrics_if_master_is_more_resent()
{
var sample1 = new NodeMetrics(_node1, 1, ImmutableHashSet.Create(
NodeMetrics.Types.Metric.Create("a", 10, Option <double> .None).Value,
NodeMetrics.Types.Metric.Create("b", 20, Option <double> .None).Value
));
var sample2 = new NodeMetrics(_node1, 0, ImmutableHashSet.Create(
NodeMetrics.Types.Metric.Create("a", 11, Option <double> .None).Value,
NodeMetrics.Types.Metric.Create("c", 30, Option <double> .None).Value
));
var merged = sample1.Merge(sample2);
merged.Timestamp.Should().Be(sample1.Timestamp);
merged.Metrics.ShouldBeEquivalentTo(sample1.Metrics);
}
/// <summary>
/// <see cref="UnarySequentialDispatcherLocalNode{TInput}"/>
/// </summary>
/// <param name="persistentCache">Persistent cache to avoid dropped data on system crash</param>
/// <param name="process">The <see cref="Task"/> to be applied to an item</param>
/// <param name="progress">Progress of the current bulk</param>
/// <param name="cts"><see cref="CancellationTokenSource"/></param>
/// <param name="circuitBreakerOptions"><see cref="CircuitBreakerOptions"/></param>
/// <param name="clusterOptions"><see cref="ClusterOptions"/></param>
/// <param name="logger"><see cref="ILogger"/></param>
public UnarySequentialDispatcherLocalNode(
IAppCache persistentCache,
Func <TInput, NodeMetrics, CancellationToken, Task> process,
IProgress <double> progress,
CancellationTokenSource cts,
CircuitBreakerOptions circuitBreakerOptions,
ClusterOptions clusterOptions,
ILogger logger) : base(
Policy.Handle <Exception>()
.AdvancedCircuitBreakerAsync(circuitBreakerOptions.CircuitBreakerFailureThreshold,
circuitBreakerOptions.CircuitBreakerSamplingDuration,
circuitBreakerOptions.CircuitBreakerMinimumThroughput,
circuitBreakerOptions.CircuitBreakerDurationOfBreak,
onBreak: (ex, timespan, context) =>
{
logger.LogError(
$"Batch processor breaker: Breaking the circuit for {timespan.TotalMilliseconds}ms due to {ex.Message}.");
},
onReset: context =>
{
logger.LogInformation(
"Batch processor breaker: Succeeded, closed the circuit.");
},
onHalfOpen: () =>
{
logger.LogWarning(
"Batch processor breaker: Half-open, next call is a trial.");
}), clusterOptions, progress, cts, logger)
{
_logger = logger;
_process = process;
_clusterOptions = clusterOptions;
ISubject <PersistentItem <TInput> > dispatcherSubject = new Subject <PersistentItem <TInput> >();
_synchronizedDispatcherSubject = Subject.Synchronize(dispatcherSubject);
_synchronizedDispatcherSubjectSubscription = _synchronizedDispatcherSubject
.ObserveOn(new EventLoopScheduler(ts => new Thread(ts)))
.Select(item =>
{
return(Observable.FromAsync(() => persistentCache.AddItemAsync(item.Entity,
item.CancellationTokenSource.Token)));
})
.Merge()
.Subscribe();
NodeMetrics = new NodeMetrics(Guid.NewGuid());
}
public void MetricGossip_should_merge_peer_metrics()
{
var m1 = new NodeMetrics(new Address("akka", "sys", "a", 2554), NewTimestamp, Collector.Sample().Metrics);
var m2 = new NodeMetrics(new Address("akka", "sys", "a", 2555), NewTimestamp, Collector.Sample().Metrics);
var g1 = MetricsGossip.Empty + m1 + m2;
g1.Nodes.Should().HaveCount(2);
var m2Updated = new NodeMetrics(m2.Address, m2.Timestamp + 1000, NewSample(m2.Metrics));
var g2 = g1 + m2Updated; // merge peers
g2.Nodes.Should().HaveCount(2);
g2.NodeMetricsFor(m1.Address).Value.Metrics.Should().BeEquivalentTo(m1.Metrics);
g2.NodeMetricsFor(m2.Address).Value.Metrics.Should().BeEquivalentTo(m2Updated.Metrics);
g2.Nodes.Where(p => p.Address.Equals(m2.Address)).ForEach(p => p.Timestamp.Should().Be(m2Updated.Timestamp));
}
public void MetricsGossip_must_add_new_NodeMetrics()
{
var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var m2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
Assert.True(m1.Metrics.Count > 3);
Assert.True(m2.Metrics.Count > 3);
var g1 = MetricsGossip.Empty + m1;
g1.Nodes.Count.ShouldBe(1);
g1.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
var g2 = g1 + m2;
g2.Nodes.Count.ShouldBe(2);
g2.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
g2.NodeMetricsFor(m2.Address).Metrics.ShouldBe(m2.Metrics);
}
public void MetricsGossip_must_merge_peer_metrics()
{
var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var m2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var g1 = MetricsGossip.Empty + m1 + m2;
g1.Nodes.Count.ShouldBe(2);
var beforeMergeNodes = g1.Nodes;
var m2Updated = m2.Copy(metrics: _collector.Sample().Metrics, timestamp: m2.Timestamp + 1000);
var g2 = g1 + m2Updated; //merge peers
g2.Nodes.Count.ShouldBe(2);
g2.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
g2.NodeMetricsFor(m2.Address).Metrics.ShouldBe(m2.Metrics);
foreach (var peer in g2.Nodes)
{
if(peer.Address == m2.Address)
peer.Timestamp.ShouldBe(m2Updated.Timestamp);
}
}
public void MetricsGossip_must_merge_an_existing_metric_set_for_a_node_and_update_node_ring()
{
var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var m2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var m3 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2556), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var m2Updated = m2.Copy(metrics: _collector.Sample().Metrics, timestamp: m2.Timestamp + 1000);
var g1 = MetricsGossip.Empty + m1 + m2;
var g2 = MetricsGossip.Empty + m3 + m2Updated;
Assert.True(g1.Nodes.All(x => x.Address == m1.Address || x.Address == m2.Address));
//should contain nodes 1,3 and the most recent version of 2
var mergedGossip = g1.Merge(g2);
XAssert.Equivalent(mergedGossip.Nodes.Select(x => x.Address),
new[] {m1.Address, m2.Address, m3.Address});
mergedGossip.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
mergedGossip.NodeMetricsFor(m2.Address).Metrics.ShouldBe(m2Updated.Metrics);
mergedGossip.NodeMetricsFor(m3.Address).Metrics.ShouldBe(m3.Metrics);
Assert.True(mergedGossip.Nodes.All(x => x.Metrics.Count > 3));
mergedGossip.NodeMetricsFor(m2.Address).Timestamp.ShouldBe(m2Updated.Timestamp);
}
public void MetricsGossip_must_get_the_current_NodeMetrics_if_it_exists_in_local_nodes()
{
var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var g1 = MetricsGossip.Empty + m1;
g1.NodeMetricsFor(m1.Address).Metrics.ShouldBe(m1.Metrics);
}
public void MetricsGossip_must_filter_nodes()
{
var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var m2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var g1 = MetricsGossip.Empty + m1 + m2;
g1.Nodes.Count.ShouldBe(2);
var g2 = g1.Filter(ImmutableHashSet.Create<Address>(new[] {m2.Address}));
g2.Nodes.Count.ShouldBe(1);
g2.Nodes.Any(x => x.Address == m1.Address).ShouldBeFalse();
g2.NodeMetricsFor(m1.Address).ShouldBe(null);
g2.NodeMetricsFor(m2.Address).Metrics.ShouldBe(m2.Metrics);
}
public void MetricsGossip_must_remove_a_node_if_it_is_no_longer_up()
{
var m1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var m2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), StandardMetrics.NewTimestamp(),
_collector.Sample().Metrics);
var g1 = MetricsGossip.Empty + m1 + m2;
g1.Nodes.Count.ShouldBe(2);
var g2 = g1.Remove(m1.Address);
g2.Nodes.Count.ShouldBe(1);
g2.Nodes.Any(x => x.Address == m1.Address).ShouldBeFalse();
g2.NodeMetricsFor(m1.Address).ShouldBe(null);
g2.NodeMetricsFor(m2.Address).Metrics.ShouldBe(m2.Metrics);
}
public MetricValuesSpec()
{
_collector = CreateMetricsCollector();
_node1 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2554), 1, _collector.Sample().Metrics);
_node2 = new NodeMetrics(new Address("akka.tcp", "sys", "a", 2555), 1, _collector.Sample().Metrics);
}