private async Task <BoolResult> UpdateClusterStateCoreAsync(OperationContext context, ClusterState clusterState)
{
(var inactiveMachineIdSet, var getUnknownMachinesResult) = await _clusterStateKey.UseReplicatedHashAsync(context, _configuration.RetryWindow, RedisOperation.UpdateClusterState, async (batch, key) =>
{
var heartbeatResultTask = CallHeartbeatAsync(context, clusterState, batch, key, MachineState.Active);
var getUnknownMachinesTask = batch.GetUnknownMachinesAsync(
key,
clusterState.MaxMachineId);
await Task.WhenAll(heartbeatResultTask, getUnknownMachinesTask);
var heartbeatResult = await heartbeatResultTask;
var getUnknownMachinesResult = await getUnknownMachinesTask;
return(heartbeatResult, getUnknownMachinesResult);
}).ThrowIfFailureAsync();
if (getUnknownMachinesResult.maxMachineId != clusterState.MaxMachineId)
{
Tracer.Debug(context, $"Retrieved unknown machines from ({clusterState.MaxMachineId}, {getUnknownMachinesResult.maxMachineId}]");
foreach (var item in getUnknownMachinesResult.unknownMachines)
{
context.LogMachineMapping(Tracer, item.Key, item.Value);
}
}
clusterState.AddUnknownMachines(getUnknownMachinesResult.maxMachineId, getUnknownMachinesResult.unknownMachines);
clusterState.SetInactiveMachines(inactiveMachineIdSet);
Tracer.Debug(context, $"Inactive machines: Count={inactiveMachineIdSet.Count}, [{string.Join(", ", inactiveMachineIdSet)}]");
Tracer.TrackMetric(context, "InactiveMachineCount", inactiveMachineIdSet.Count);
foreach (var machineMapping in clusterState.LocalMachineMappings)
{
if (!clusterState.TryResolveMachineId(machineMapping.Location, out var machineId))
{
return(new BoolResult($"Invalid redis cluster state on machine {machineMapping}. (Missing location {machineMapping.Location})"));
}
else if (machineId != machineMapping.Id)
{
Tracer.Warning(context, $"Machine id mismatch for location {machineMapping.Location}. Registered id: {machineMapping.Id}. Cluster state id: {machineId}. Updating registered id with cluster state id.");
machineMapping.Id = machineId;
}
if (getUnknownMachinesResult.maxMachineId < machineMapping.Id.Index)
{
return(new BoolResult($"Invalid redis cluster state on machine {machineMapping} (redis max machine id={getUnknownMachinesResult.maxMachineId})"));
}
}
return(BoolResult.Success);
}
private async Task <BoolResult> UpdateLocalClusterStateAsync(OperationContext context, ClusterState clusterState)
{
(var heartbeatResult, var getUnknownMachinesResult) = await _clusterStateKey.UseReplicatedHashAsync(context, _configuration.RetryWindow, RedisOperation.UpdateClusterState, async (batch, key) =>
{
var heartbeatResultTask = CallHeartbeatAsync(context, batch, key, MachineState.Active);
var getUnknownMachinesTask = batch.GetUnknownMachinesAsync(
key,
clusterState.MaxMachineId);
await Task.WhenAll(heartbeatResultTask, getUnknownMachinesTask);
var heartbeatResult = await heartbeatResultTask;
var getUnknownMachinesResult = await getUnknownMachinesTask;
return(heartbeatResult, getUnknownMachinesResult);
}).ThrowIfFailureAsync();
if (getUnknownMachinesResult.maxMachineId < LocalMachineId.Index)
{
return(new BoolResult($"Invalid redis cluster state on machine {LocalMachineId} (redis max machine id={getUnknownMachinesResult.maxMachineId})"));
}
if (heartbeatResult.priorState == MachineState.Unavailable || heartbeatResult.priorState == MachineState.Expired)
{
clusterState.LastInactiveTime = _clock.UtcNow;
}
if (getUnknownMachinesResult.maxMachineId != clusterState.MaxMachineId)
{
Tracer.Debug(context, $"Retrieved unknown machines from ({clusterState.MaxMachineId}, {getUnknownMachinesResult.maxMachineId}]");
foreach (var item in getUnknownMachinesResult.unknownMachines)
{
context.LogMachineMapping(Tracer, item.Key, item.Value);
}
}
clusterState.AddUnknownMachines(getUnknownMachinesResult.maxMachineId, getUnknownMachinesResult.unknownMachines);
clusterState.SetInactiveMachines(heartbeatResult.inactiveMachineIdSet);
Tracer.Debug(context, $"Inactive machines: Count={heartbeatResult.inactiveMachineIdSet.Count}, [{string.Join(", ", heartbeatResult.inactiveMachineIdSet)}]");
Tracer.TrackMetric(context, "InactiveMachineCount", heartbeatResult.inactiveMachineIdSet.Count);
return(BoolResult.Success);
}
private Task <Result <HashEntry[]> > UpdateLocalClusterStateAsync(OperationContext context, ClusterState clusterState, RedisDatabaseAdapter redisDb)
{
return(redisDb.ExecuteBatchAsync(context, async batch =>
{
var heartbeatResultTask = CallHeartbeatAsync(context, batch, MachineState.Active);
var getUnknownMachinesTask = batch.GetUnknownMachinesAsync(
_clusterStateKey,
clusterState.MaxMachineId);
// Only master should mirror cluster state
bool shouldMirrorClusterState = _role == Role.Master &&
HasSecondary &&
_configuration.MirrorClusterState
// Only mirror after a long interval, but not long enough to allow machines to appear expired
&& !_lastClusterStateMirrorTime.IsRecent(_clock.UtcNow, _configuration.ClusterStateMirrorInterval)
// Only mirror from primary to secondary, so no need to dump cluster state if this is the secondary
&& IsPrimary(redisDb);
Task <HashEntry[]> dumpClusterStateBlobTask = shouldMirrorClusterState
? batch.AddOperation(_clusterStateKey, b => b.HashGetAllAsync(_clusterStateKey))
: _emptyClusterStateDump;
await Task.WhenAll(heartbeatResultTask, getUnknownMachinesTask, dumpClusterStateBlobTask);
var clusterStateBlob = await dumpClusterStateBlobTask ?? CollectionUtilities.EmptyArray <HashEntry>();
var heartbeatResult = await heartbeatResultTask;
var getUnknownMachinesResult = await getUnknownMachinesTask;
if (shouldMirrorClusterState)
{
_lastClusterStateMirrorTime = _clock.UtcNow;
}
if (getUnknownMachinesResult.maxMachineId < LocalMachineId.Index)
{
return Result.FromErrorMessage <HashEntry[]>($"Invalid {GetDbName(redisDb)} redis cluster state on machine {LocalMachineId} (max machine id={getUnknownMachinesResult.maxMachineId})");
}
if (heartbeatResult.priorState == MachineState.Unavailable || heartbeatResult.priorState == MachineState.Expired)
{
clusterState.LastInactiveTime = _clock.UtcNow;
}
if (getUnknownMachinesResult.maxMachineId != clusterState.MaxMachineId)
{
Tracer.Debug(context, $"Retrieved unknown machines from ({clusterState.MaxMachineId}, {getUnknownMachinesResult.maxMachineId}]");
foreach (var item in getUnknownMachinesResult.unknownMachines)
{
context.LogMachineMapping(Tracer, item.Key, item.Value);
}
}
clusterState.AddUnknownMachines(getUnknownMachinesResult.maxMachineId, getUnknownMachinesResult.unknownMachines);
clusterState.SetInactiveMachines(heartbeatResult.inactiveMachineIdSet);
Tracer.Debug(context, $"Inactive machines: Count={heartbeatResult.inactiveMachineIdSet.Count}, [{string.Join(", ", heartbeatResult.inactiveMachineIdSet)}]");
Tracer.TrackMetric(context, "InactiveMachineCount", heartbeatResult.inactiveMachineIdSet.Count);
return Result.Success(await dumpClusterStateBlobTask ?? CollectionUtilities.EmptyArray <HashEntry>());
},
RedisOperation.UpdateClusterState));
}