public Task<Result<MachineMapping>> RegisterMachineAsync(OperationContext context, MachineLocation machineLocation)
{
return context.PerformOperationAsync(Tracer, async () =>
{
if (Configuration.DistributedContentConsumerOnly)
{
return Result.Success(new MachineMapping(machineLocation, new MachineId(0)));
}
// Get the local machine id
var machineIdAndIsAdded = await _clusterStateKey.UseNonConcurrentReplicatedHashAsync(
context,
Configuration.RetryWindow,
RedisOperation.StartupGetOrAddLocalMachine,
(batch, key) => batch.GetOrAddMachineAsync(key, machineLocation.ToString(), _clock.UtcNow),
timeout: Configuration.ClusterRedisOperationTimeout)
.ThrowIfFailureAsync();
Tracer.Debug(context, $"Assigned machine id={machineIdAndIsAdded.machineId}, location={machineLocation}, isAdded={machineIdAndIsAdded.isAdded}.");
return Result.Success(new MachineMapping(machineLocation, new MachineId(machineIdAndIsAdded.machineId)));
},
traceOperationStarted: false,
extraEndMessage: r =>
{
if (r.Succeeded)
{
return $"MachineLocation=[{r.Value.Location}] MachineId=[{r.Value.Id}]";
}
else
{
return $"MachineLocation=[{machineLocation}]";
}
});
}
public async Task <MachineMapping> RegisterMachineAsync(OperationContext context, MachineLocation machineLocation)
{
// Get the local machine id
var machineIdAndIsAdded = await _clusterStateKey.UseNonConcurrentReplicatedHashAsync(
context,
_configuration.RetryWindow,
RedisOperation.StartupGetOrAddLocalMachine,
(batch, key) => batch.GetOrAddMachineAsync(key, machineLocation.ToString(), _clock.UtcNow),
timeout : _configuration.ClusterRedisOperationTimeout)
.ThrowIfFailureAsync();
Tracer.Debug(context, $"Assigned machine id={machineIdAndIsAdded.machineId}, location={machineLocation}, isAdded={machineIdAndIsAdded.isAdded}.");
return(new MachineMapping(machineLocation, new MachineId(machineIdAndIsAdded.machineId)));
}
/// <inheritdoc />
public Task <Result <CheckpointState> > GetCheckpointStateAsync(OperationContext context)
{
return(context.PerformOperationAsync(
Tracer,
async() =>
{
var(checkpoints, startCursor) = await _checkpointsKey.UseNonConcurrentReplicatedHashAsync(
context,
Configuration.RetryWindow,
RedisOperation.GetCheckpoint,
(batch, key) => batch.GetCheckpointsInfoAsync(key, _clock.UtcNow),
timeout: Configuration.ClusterRedisOperationTimeout)
.ThrowIfFailureAsync();
var roleResult = await UpdateRoleAsync(context, release: false);
if (!roleResult.Succeeded)
{
return new ErrorResult(roleResult).AsResult <Result <CheckpointState> >();
}
_role = roleResult.Value;
var masterName = await _masterLeaseKey.UseNonConcurrentReplicatedHashAsync(
context,
Configuration.RetryWindow,
RedisOperation.GetCheckpoint,
(batch, key) => batch.AddOperation("GetRole", b => b.HashGetAsync(key, "M#1.MachineName")),
timeout: Configuration.ClusterRedisOperationTimeout)
.ThrowIfFailureAsync();
var masterLocation = masterName.IsNull ? default(MachineLocation) : new MachineLocation((string)masterName);
var maxCheckpoint = checkpoints.MaxByOrDefault(c => c.CheckpointCreationTime);
if (maxCheckpoint == null)
{
Tracer.Debug(context, $"Getting checkpoint state: Can't find a checkpoint: Start cursor time: {startCursor}");
// Add slack for start cursor to account for clock skew between event hub and redis
var epochStartCursor = startCursor - Configuration.EventStore.NewEpochEventStartCursorDelay;
return CheckpointState.CreateUnavailable(_role.Value, epochStartCursor, masterLocation);
}
Tracer.Debug(context, $"Getting checkpoint state: Found checkpoint '{maxCheckpoint}'");
return Result.Success(new CheckpointState(_role.Value, new EventSequencePoint(maxCheckpoint.SequenceNumber), maxCheckpoint.CheckpointId, maxCheckpoint.CheckpointCreationTime, new MachineLocation(maxCheckpoint.MachineName), masterLocation));
},
Counters[GlobalStoreCounters.GetCheckpointState]));
}
private async Task <Result <Role> > UpdateRoleAsync(OperationContext context, bool release)
{
return(await context.PerformOperationAsync <Result <Role> >(
Tracer,
async() =>
{
// This mutex ensure that Release of master role during shutdown and Heartbeat role acquisition are synchronized.
// Ensuring that a heartbeat during shutdown doesn't trigger the released master role to be acquired again.
using (await _roleMutex.AcquireAsync())
{
if (ShutdownStarted)
{
// Don't acquire a role during shutdown
return Role.Worker;
}
var configuredRole = _configuration.Checkpoint?.Role;
if (configuredRole != null)
{
return configuredRole.Value;
}
var localMachineName = _configuration.PrimaryMachineLocation.ToString();
var masterAcquisitonResult = await _masterLeaseKey.UseNonConcurrentReplicatedHashAsync(context, _configuration.RetryWindow, RedisOperation.UpdateRole, (batch, key) => batch.AcquireMasterRoleAsync(
masterRoleRegistryKey: key,
machineName: localMachineName,
currentTime: _clock.UtcNow,
leaseExpiryTime: _configuration.Checkpoint.MasterLeaseExpiryTime,
// 1 master only is allowed. This should be changed if more than one master becomes a possible configuration
slotCount: 1,
release: release
),
timeout: _configuration.ClusterRedisOperationTimeout).ThrowIfFailureAsync();
if (release)
{
Tracer.Debug(context, $"'{localMachineName}' released master role.");
return Role.Worker;
}
if (masterAcquisitonResult != null)
{
var priorMachineName = masterAcquisitonResult.Value.PriorMasterMachineName;
if (priorMachineName != localMachineName || masterAcquisitonResult.Value.PriorMachineStatus != SlotStatus.Acquired)
{
Tracer.Debug(context, $"'{localMachineName}' acquired master role from '{priorMachineName}', Status: '{masterAcquisitonResult?.PriorMachineStatus}', LastHeartbeat: '{masterAcquisitonResult?.PriorMasterLastHeartbeat}'");
}
return Role.Master;
}
else
{
return Role.Worker;
}
}
},
Counters[GlobalStoreCounters.UpdateRole]));
}