internal async Task <int> RegisterMachineAsync(OperationContext context, MachineLocation machineLocation)
{
// Get the local machine id
var machineIdAndIsAdded = await _clusterStateKey.UseReplicatedHashAsync(
context,
RedisOperation.StartupGetOrAddLocalMachine,
(batch, key) => batch.GetOrAddMachineAsync(key, machineLocation.ToString(), _clock.UtcNow))
.ThrowIfFailureAsync();
Tracer.Debug(context, $"Assigned machine id={machineIdAndIsAdded.machineId}, location={machineLocation}, isAdded={machineIdAndIsAdded.isAdded}.");
return(machineIdAndIsAdded.machineId);
}
private async Task <Result <Role> > UpdateRoleAsync(OperationContext context, bool release)
{
return(await context.PerformOperationAsync <Result <Role> >(
Tracer,
async() =>
{
// This mutex ensure that Release of master role during shutdown and Heartbeat role acquisition are synchronized.
// Ensuring that a heartbeat during shutdown doesn't trigger the released master role to be acquired again.
using (await _roleMutex.AcquireAsync())
{
if (ShutdownStarted)
{
// Don't acquire a role during shutdown
return Role.Worker;
}
var configuredRole = _configuration.Checkpoint?.Role;
if (configuredRole != null)
{
return configuredRole.Value;
}
var localMachineName = _configuration.PrimaryMachineLocation.ToString();
var masterAcquisitonResult = await _masterLeaseKey.UseReplicatedHashAsync(context, _configuration.RetryWindow, RedisOperation.UpdateRole, (batch, key) => batch.AcquireMasterRoleAsync(
masterRoleRegistryKey: key,
machineName: localMachineName,
currentTime: _clock.UtcNow,
leaseExpiryTime: _configuration.Checkpoint.MasterLeaseExpiryTime,
// 1 master only is allowed. This should be changed if more than one master becomes a possible configuration
slotCount: 1,
release: release
)).ThrowIfFailureAsync();
if (release)
{
Tracer.Debug(context, $"'{localMachineName}' released master role.");
return Role.Worker;
}
if (masterAcquisitonResult != null)
{
var priorMachineName = masterAcquisitonResult.Value.PriorMasterMachineName;
if (priorMachineName != localMachineName || masterAcquisitonResult.Value.PriorMachineStatus != SlotStatus.Acquired)
{
Tracer.Debug(context, $"'{localMachineName}' acquired master role from '{priorMachineName}', Status: '{masterAcquisitonResult?.PriorMachineStatus}', LastHeartbeat: '{masterAcquisitonResult?.PriorMasterLastHeartbeat}'");
}
return Role.Master;
}
else
{
return Role.Worker;
}
}
},
Counters[GlobalStoreCounters.UpdateRole]));
}
/// <inheritdoc />
public Task <Result <CheckpointState> > GetCheckpointStateAsync(OperationContext context)
{
return(context.PerformOperationAsync(
Tracer,
async() =>
{
var(checkpoints, startCursor) = await _checkpointsKey.UseReplicatedHashAsync(
context,
_configuration.RetryWindow,
RedisOperation.GetCheckpoint,
(batch, key) => batch.GetCheckpointsInfoAsync(key, _clock.UtcNow))
.ThrowIfFailureAsync();
var roleResult = await UpdateRoleAsync(context, release: false);
if (!roleResult.Succeeded)
{
return new ErrorResult(roleResult).AsResult <Result <CheckpointState> >();
}
_role = roleResult.Value;
var maxCheckpoint = checkpoints.MaxByOrDefault(c => c.CheckpointCreationTime);
if (maxCheckpoint == null)
{
Tracer.Debug(context, $"Getting checkpoint state: Can't find a checkpoint: Start cursor time: {startCursor}");
// Add slack for start cursor to account for clock skew between event hub and redis
var epochStartCursor = startCursor - _configuration.EventStore.NewEpochEventStartCursorDelay;
return CheckpointState.CreateUnavailable(_role.Value, epochStartCursor);
}
Tracer.Debug(context, $"Getting checkpoint state: Found checkpoint '{maxCheckpoint}'");
return Result.Success(new CheckpointState(_role.Value, new EventSequencePoint(maxCheckpoint.SequenceNumber), maxCheckpoint.CheckpointId, maxCheckpoint.CheckpointCreationTime, new MachineLocation(maxCheckpoint.MachineName)));
},
Counters[GlobalStoreCounters.GetCheckpointState],
// Using a timeout to make sure the operation finishes: this is important because we want for heartbeat operations
// that call this method to keep running to avoid stale checkpoints.
timeout: Configuration.GetCheckpointStateTimeout));
}