private async Task TryMirrorRedisHashDataAsync(OperationContext context, RedisDatabaseAdapter source, RedisDatabaseAdapter target, long?postMirrorSourceVersion = null)
{
await context.PerformOperationAsync(
_host.Tracer,
async() =>
{
var sourceDump = await source.ExecuteBatchAsync(context,
b => b.AddOperation(_key, b => b.KeyDumpAsync(_key)),
RedisOperation.HashGetKeys);
await target.ExecuteBatchAsync(context,
b =>
{
var deleteTask = b.AddOperation(_key, b => b.KeyDeleteAsync(_key));
var restoreTask = b.AddOperation(_key, b => b.KeyRestoreAsync(_key, sourceDump).WithResultAsync(Unit.Void));
return(Task.WhenAll(deleteTask, restoreTask).WithResultAsync(Unit.Void));
},
RedisOperation.HashDeleteAndRestore);
if (postMirrorSourceVersion.HasValue)
{
await source.ExecuteBatchAsync(context,
b => b.AddOperation(_key, b => b.HashSetAsync(_key, nameof(ReplicatedHashVersionNumber), postMirrorSourceVersion.Value)),
RedisOperation.HashSetValue);
}
return(Result.Success(sourceDump.Length));
},
extraStartMessage : $"({_redis.GetDbName(source)} -> {_redis.GetDbName(target)}) Key={_key}, PostMirrorSourceVersion={postMirrorSourceVersion ?? -1L}",
extraEndMessage : r => $"({_redis.GetDbName(source)} -> {_redis.GetDbName(target)}) Key={_key}, Length={r.GetValueOrDefault(-1)}").IgnoreFailure();
}
private Task <bool> CompareExchangeInternalAsync(OperationContext context, StrongFingerprint strongFingerprint, string expectedReplacementToken, ContentHashListWithDeterminism expected, ContentHashListWithDeterminism replacement, string newReplacementToken)
{
var key = GetKey(strongFingerprint.WeakFingerprint);
var replacementMetadata = new MetadataEntry(replacement, _clock.UtcNow);
var replacementBytes = SerializeMetadataEntry(replacementMetadata);
byte[] selectorBytes = SerializeSelector(strongFingerprint.Selector, isReplacementToken: false);
byte[] tokenFieldNameBytes = SerializeSelector(strongFingerprint.Selector, isReplacementToken: true);
return(_redis.ExecuteBatchAsync(context, batch =>
{
var task = batch.CompareExchangeAsync(key, selectorBytes, tokenFieldNameBytes, expectedReplacementToken, replacementBytes, newReplacementToken);
batch.KeyExpireAsync(key, _metadataExpiryTime).FireAndForget(context);
return task;
}, RedisOperation.CompareExchange));
}
private async Task <Result <Role> > UpdateRoleAsync(OperationContext context, bool release)
{
return(await context.PerformOperationAsync <Result <Role> >(
Tracer,
async() =>
{
// This mutex ensure that Release of master role during shutdown and Heartbeat role acquisition are synchronized.
// Ensuring that a heartbeat during shutdown doesn't trigger the released master role to be acquired again.
using (await _roleMutex.AcquireAsync())
{
if (ShutdownStarted)
{
// Don't acquire a role during shutdown
return Role.Worker;
}
var configuredRole = _configuration.Checkpoint?.Role;
if (configuredRole != null)
{
return configuredRole.Value;
}
var localMachineName = LocalMachineLocation.ToString();
var masterAcquisitonResult = await _primaryRedisDb.ExecuteBatchAsync(context, batch => batch.AcquireMasterRoleAsync(
masterRoleRegistryKey: _masterLeaseKey,
machineName: localMachineName,
currentTime: _clock.UtcNow,
leaseExpiryTime: _configuration.Checkpoint.MasterLeaseExpiryTime,
// 1 master only is allowed. This should be changed if more than one master becomes a possible configuration
slotCount: 1,
release: release
), RedisOperation.UpdateRole);
if (release)
{
Tracer.Debug(context, $"'{localMachineName}' released master role.");
return Role.Worker;
}
if (masterAcquisitonResult != null)
{
var priorMachineName = masterAcquisitonResult.Value.PriorMasterMachineName;
if (priorMachineName != localMachineName || masterAcquisitonResult.Value.PriorMachineStatus != SlotStatus.Acquired)
{
Tracer.Debug(context, $"'{localMachineName}' acquired master role from '{priorMachineName}', Status: '{masterAcquisitonResult?.PriorMachineStatus}', LastHeartbeat: '{masterAcquisitonResult?.PriorMasterLastHeartbeat}'");
}
return Role.Master;
}
else
{
return Role.Worker;
}
}
},
Counters[GlobalStoreCounters.UpdateRole]).FireAndForgetAndReturnTask(context));
}
/// <inheritdoc />
public Task <BoolResult> UpdateClusterStateAsync(OperationContext context, ClusterState clusterState)
{
return(context.PerformOperationAsync(
Tracer,
async() =>
{
HashEntry[] clusterStateDump = await ExecuteRedisFallbackAsync(context, redisDb => UpdateLocalClusterStateAsync(context, clusterState, redisDb)).ThrowIfFailureAsync();
if (clusterStateDump.Length != 0 && HasSecondary && _configuration.MirrorClusterState)
{
Tracer.Debug(context, $"Mirroring cluster state with '{clusterStateDump.Length}' entries to secondary");
await _secondaryRedisDb.ExecuteBatchAsync(context, batch => batch.AddOperation(_clusterStateKey, async b =>
{
await b.HashSetAsync(_clusterStateKey, clusterStateDump);
return Unit.Void;
}),
RedisOperation.MirrorClusterState).FireAndForgetErrorsAsync(context);
}
return BoolResult.Success;
}, Counters[GlobalStoreCounters.UpdateClusterState]));
}
private Task <Result <HashEntry[]> > UpdateLocalClusterStateAsync(OperationContext context, ClusterState clusterState, RedisDatabaseAdapter redisDb)
{
return(redisDb.ExecuteBatchAsync(context, async batch =>
{
var heartbeatResultTask = CallHeartbeatAsync(context, batch, MachineState.Active);
var getUnknownMachinesTask = batch.GetUnknownMachinesAsync(
_clusterStateKey,
clusterState.MaxMachineId);
// Only master should mirror cluster state
bool shouldMirrorClusterState = _role == Role.Master &&
HasSecondary &&
_configuration.MirrorClusterState
// Only mirror after a long interval, but not long enough to allow machines to appear expired
&& !_lastClusterStateMirrorTime.IsRecent(_clock.UtcNow, _configuration.ClusterStateMirrorInterval)
// Only mirror from primary to secondary, so no need to dump cluster state if this is the secondary
&& IsPrimary(redisDb);
Task <HashEntry[]> dumpClusterStateBlobTask = shouldMirrorClusterState
? batch.AddOperation(_clusterStateKey, b => b.HashGetAllAsync(_clusterStateKey))
: _emptyClusterStateDump;
await Task.WhenAll(heartbeatResultTask, getUnknownMachinesTask, dumpClusterStateBlobTask);
var clusterStateBlob = await dumpClusterStateBlobTask ?? CollectionUtilities.EmptyArray <HashEntry>();
var heartbeatResult = await heartbeatResultTask;
var getUnknownMachinesResult = await getUnknownMachinesTask;
if (shouldMirrorClusterState)
{
_lastClusterStateMirrorTime = _clock.UtcNow;
}
if (getUnknownMachinesResult.maxMachineId < LocalMachineId.Index)
{
return Result.FromErrorMessage <HashEntry[]>($"Invalid {GetDbName(redisDb)} redis cluster state on machine {LocalMachineId} (max machine id={getUnknownMachinesResult.maxMachineId})");
}
if (heartbeatResult.priorState == MachineState.Unavailable || heartbeatResult.priorState == MachineState.Expired)
{
clusterState.LastInactiveTime = _clock.UtcNow;
}
if (getUnknownMachinesResult.maxMachineId != clusterState.MaxMachineId)
{
Tracer.Debug(context, $"Retrieved unknown machines from ({clusterState.MaxMachineId}, {getUnknownMachinesResult.maxMachineId}]");
foreach (var item in getUnknownMachinesResult.unknownMachines)
{
context.LogMachineMapping(Tracer, item.Key, item.Value);
}
}
clusterState.AddUnknownMachines(getUnknownMachinesResult.maxMachineId, getUnknownMachinesResult.unknownMachines);
clusterState.SetInactiveMachines(heartbeatResult.inactiveMachineIdSet);
Tracer.Debug(context, $"Inactive machines: Count={heartbeatResult.inactiveMachineIdSet.Count}, [{string.Join(", ", heartbeatResult.inactiveMachineIdSet)}]");
Tracer.TrackMetric(context, "InactiveMachineCount", heartbeatResult.inactiveMachineIdSet.Count);
return Result.Success(await dumpClusterStateBlobTask ?? CollectionUtilities.EmptyArray <HashEntry>());
},
RedisOperation.UpdateClusterState));
}
