private async Task <BoolResult> UpdateClusterStateCoreAsync(OperationContext context, ClusterState clusterState, MachineState machineState)
{
(var inactiveMachineIdSet, var closedMachineIdSet, var getUnknownMachinesResult) = await _clusterStateKey.UseNonConcurrentReplicatedHashAsync(
context, _configuration.RetryWindow, RedisOperation.UpdateClusterState, async (batch, key) =>
{
var heartbeatResultTask = CallHeartbeatAsync(context, clusterState, batch, key, machineState);
var getUnknownMachinesTask = batch.GetUnknownMachinesAsync(
key,
clusterState.MaxMachineId);
await Task.WhenAll(heartbeatResultTask, getUnknownMachinesTask);
var heartbeatResult = await heartbeatResultTask;
var getUnknownMachinesResult = await getUnknownMachinesTask;
return(heartbeatResult.inactiveMachineIdSet, heartbeatResult.closedMachineIdSet, getUnknownMachinesResult);
},
timeout : _configuration.ClusterRedisOperationTimeout).ThrowIfFailureAsync();
Contract.Assert(inactiveMachineIdSet != null, "inactiveMachineIdSet != null");
Contract.Assert(closedMachineIdSet != null, "closedMachineIdSet != null");
if (getUnknownMachinesResult.maxMachineId != clusterState.MaxMachineId)
{
Tracer.Debug(context, $"Retrieved unknown machines from ({clusterState.MaxMachineId}, {getUnknownMachinesResult.maxMachineId}]");
foreach (var item in getUnknownMachinesResult.unknownMachines)
{
context.LogMachineMapping(Tracer, item.Key, item.Value);
}
}
clusterState.AddUnknownMachines(getUnknownMachinesResult.maxMachineId, getUnknownMachinesResult.unknownMachines);
clusterState.SetMachineStates(inactiveMachineIdSet, closedMachineIdSet).ThrowIfFailure();
Tracer.Debug(context, $"Inactive machines: Count={inactiveMachineIdSet.Count}, [{string.Join(", ", inactiveMachineIdSet)}]");
Tracer.TrackMetric(context, "InactiveMachineCount", inactiveMachineIdSet.Count);
foreach (var machineMapping in clusterState.LocalMachineMappings)
{
if (!clusterState.TryResolveMachineId(machineMapping.Location, out var machineId))
{
return(new BoolResult($"Invalid redis cluster state on machine {machineMapping}. (Missing location {machineMapping.Location})"));
}
else if (machineId != machineMapping.Id)
{
Tracer.Warning(context, $"Machine id mismatch for location {machineMapping.Location}. Registered id: {machineMapping.Id}. Cluster state id: {machineId}. Updating registered id with cluster state id.");
machineMapping.Id = machineId;
}
if (getUnknownMachinesResult.maxMachineId < machineMapping.Id.Index)
{
return(new BoolResult($"Invalid redis cluster state on machine {machineMapping} (redis max machine id={getUnknownMachinesResult.maxMachineId})"));
}
}
return(BoolResult.Success);
}
private async Task <Result <MachineState> > UpdateClusterStateCoreAsync(
OperationContext context,
ClusterState clusterState,
MachineState machineState)
{
var heartbeatResponse = await CallHeartbeatAsync(context, clusterState, machineState);
var updates = await _storage.GetClusterUpdatesAsync(context, new GetClusterUpdatesRequest()
{
MaxMachineId = clusterState.MaxMachineId
}).ThrowIfFailureAsync();
BitMachineIdSet inactiveMachineIdSet = heartbeatResponse.InactiveMachines;
BitMachineIdSet closedMachineIdSet = heartbeatResponse.ClosedMachines;
Contract.Assert(inactiveMachineIdSet != null, "inactiveMachineIdSet != null");
Contract.Assert(closedMachineIdSet != null, "closedMachineIdSet != null");
if (updates.MaxMachineId != clusterState.MaxMachineId)
{
Tracer.Debug(context, $"Retrieved unknown machines from ({clusterState.MaxMachineId}, {updates.MaxMachineId}]");
if (updates.UnknownMachines != null)
{
foreach (var item in updates.UnknownMachines)
{
context.LogMachineMapping(Tracer, item.Key, item.Value);
}
}
}
if (updates.UnknownMachines != null)
{
clusterState.AddUnknownMachines(updates.MaxMachineId, updates.UnknownMachines);
}
clusterState.SetMachineStates(inactiveMachineIdSet, closedMachineIdSet).ThrowIfFailure();
Tracer.Debug(context, $"Inactive machines: Count={inactiveMachineIdSet.Count}, [{string.Join(", ", inactiveMachineIdSet)}]");
Tracer.TrackMetric(context, "InactiveMachineCount", inactiveMachineIdSet.Count);
if (!_configuration.DistributedContentConsumerOnly)
{
foreach (var machineMapping in clusterState.LocalMachineMappings)
{
if (!clusterState.TryResolveMachineId(machineMapping.Location, out var machineId))
{
return(Result.FromErrorMessage <MachineState>($"Invalid cluster state on machine {machineMapping}. (Missing location {machineMapping.Location})"));
}
else if (machineId != machineMapping.Id)
{
Tracer.Warning(context, $"Machine id mismatch for location {machineMapping.Location}. Registered id: {machineMapping.Id}. Cluster state id: {machineId}. Updating registered id with cluster state id.");
machineMapping.Id = machineId;
}
if (updates.MaxMachineId < machineMapping.Id.Index)
{
return(Result.FromErrorMessage <MachineState>($"Invalid cluster state on machine {machineMapping} (max machine id={updates.MaxMachineId})"));
}
}
}
return(heartbeatResponse.PriorState);
}