private async Task NotifyStatusesAsync()
{
var statuses = await _client.Health.Service(_clusterName, null, false, new QueryOptions
{
WaitIndex = _index,
WaitTime = _blockingWaitTime
});
_index = statuses.LastIndex;
var kvKey = _clusterName + "/";
var kv = await _client.KV.List(kvKey);
var memberIds = new Dictionary <string, long>();
foreach (var v in kv.Response)
{
//Read the ID per member.
//The value is used to see if an existing node have changed its ID over time
//meaning that it has Re-joined the cluster.
memberIds[v.Key] = BitConverter.ToInt64(v.Value, 0);
}
var memberStatuses =
from v in statuses.Response
let memberIdKey = $"{_clusterName}/{v.Service.Address}:{v.Service.Port}"
let memberId = memberIds[memberIdKey]
let passing = Equals(v.Checks[1].Status, HealthStatus.Passing)
select new MemberStatus(memberId, v.Service.Address, v.Service.Port, v.Service.Tags, passing);
var res = new ClusterTopologyEvent(memberStatuses);
Actor.EventStream.Publish(res);
}
private void NotifyStatuses()
{
MemberStatus status = null;
if (_isServer)
{
status = new MemberStatus(_serverAddress, _serverHost, _serverPort, _kinds, true, _okStatus);
}
else
{
var responder = Remote.Remote.SpawnNamedAsync(_serverAddress, KINDS_RESPONDER, KINDS_RESPONDER, _timeout).Result;
if (responder.Pid != null)
{
try
{
var response = RootContext.Empty.RequestAsync <GetKindsResponse>(responder.Pid, new GetKinds(), _timeout).Result;
status = new MemberStatus(_serverAddress, _serverHost, _serverPort, response.Kinds, true, _okStatus);
}
catch (Exception ex) when(ex is TimeoutException || ex.InnerException is TimeoutException)
{
status = new MemberStatus(_serverAddress, _serverHost, _serverPort, new string[0], true, _koStatus);
}
}
else
{
status = new MemberStatus(_serverAddress, _serverHost, _serverPort, new string[0], false, _koStatus);
}
}
var @event = new ClusterTopologyEvent(new[] { status });
Actor.EventStream.Publish(@event);
Thread.Sleep(TimeSpan.FromMinutes(1));
}
private async Task NotifyStatuses(ulong index, PID self)
{
var statuses = await _client.Health.Service(
_clusterName, null, false, new QueryOptions
{
WaitIndex = index,
WaitTime = _options.BlockingWaitTime
}
);
Logger.LogDebug("Consul response: {@Response}", (object)statuses.Response);
var reportedServices =
statuses.Response
.Select(
x => new
{
Status = new MemberStatus(
x.Service.ID, x.Service.Address, x.Service.Port, x.Service.Tags,
x.Checks.All(c => c.Status.Status != "critical"),
_statusValueSerializer.Deserialize(x.Service.Meta["StatusValue"])
),
DeregisterInterval = Parse(x.Service.Meta["DeregisterInterval"])
}
)
.ToList();
var memberStatuses = reportedServices.Select(x => x.Status).ToList();
// Update Tags (kinds) for this member
_kinds = memberStatuses.FirstOrDefault(x => x.Address == _address && x.Port == _port)?.Kinds.ToArray();
// Consul has the minimal deregister interval of 1 minute. Our default interval is 30 seconds, but Consul won't support it.
// Therefore, we check if there are any members that fails on TTL check is recorded and when the specified interval expires,
OverrideConsulDeregisterInterval();
var res = new ClusterTopologyEvent(memberStatuses);
_system.EventStream.Publish(res);
_system.Root.Send(self, new CheckStatus {
Index = statuses.LastIndex
});
void OverrideConsulDeregisterInterval()
{
var deadMembers = reportedServices
.Where(x => !x.Status.Alive && x.DeregisterInterval != default)
.Select(x => new DeadMember(x.Status.MemberId, DateTimeOffset.UtcNow, x.DeregisterInterval))
.ToList();
if (deadMembers.Count == 0)
{
return;
}
// Remove members that have recovered
_knownDeadMembers.RemoveAll(x => !deadMembers.Contains(x));
// Find newly discovered dead members
var newDeadMembers = deadMembers.Except(_knownDeadMembers);
// Add newly discovered dead members to the list of known dead members.
// Cannot use Concat/Distinct since we must keep the timestamp.
_knownDeadMembers.AddRange(newDeadMembers);
var reportMissing = _knownDeadMembers
.Where(x => DateTimeOffset.UtcNow > x.FoundDeadAt + x.DeregisterInterval)
.Select(x => x.MemberId)
.ToList();
if (reportMissing.Count > 0)
{
Logger.LogInformation("Members to be excluded {@Dead}", reportMissing);
}
memberStatuses.RemoveAll(x => reportMissing.Contains(x.MemberId));
}
