internal async Task <int> RegisterMachineAsync(OperationContext context, MachineLocation machineLocation) { // Get the local machine id var machineIdAndIsAdded = await _clusterStateKey.UseReplicatedHashAsync( context, RedisOperation.StartupGetOrAddLocalMachine, (batch, key) => batch.GetOrAddMachineAsync(key, machineLocation.ToString(), _clock.UtcNow)) .ThrowIfFailureAsync(); Tracer.Debug(context, $"Assigned machine id={machineIdAndIsAdded.machineId}, location={machineLocation}, isAdded={machineIdAndIsAdded.isAdded}."); return(machineIdAndIsAdded.machineId); }
private async Task <Result <Role> > UpdateRoleAsync(OperationContext context, bool release) { return(await context.PerformOperationAsync <Result <Role> >( Tracer, async() => { // This mutex ensure that Release of master role during shutdown and Heartbeat role acquisition are synchronized. // Ensuring that a heartbeat during shutdown doesn't trigger the released master role to be acquired again. using (await _roleMutex.AcquireAsync()) { if (ShutdownStarted) { // Don't acquire a role during shutdown return Role.Worker; } var configuredRole = _configuration.Checkpoint?.Role; if (configuredRole != null) { return configuredRole.Value; } var localMachineName = _configuration.PrimaryMachineLocation.ToString(); var masterAcquisitonResult = await _masterLeaseKey.UseReplicatedHashAsync(context, _configuration.RetryWindow, RedisOperation.UpdateRole, (batch, key) => batch.AcquireMasterRoleAsync( masterRoleRegistryKey: key, machineName: localMachineName, currentTime: _clock.UtcNow, leaseExpiryTime: _configuration.Checkpoint.MasterLeaseExpiryTime, // 1 master only is allowed. This should be changed if more than one master becomes a possible configuration slotCount: 1, release: release )).ThrowIfFailureAsync(); if (release) { Tracer.Debug(context, $"'{localMachineName}' released master role."); return Role.Worker; } if (masterAcquisitonResult != null) { var priorMachineName = masterAcquisitonResult.Value.PriorMasterMachineName; if (priorMachineName != localMachineName || masterAcquisitonResult.Value.PriorMachineStatus != SlotStatus.Acquired) { Tracer.Debug(context, $"'{localMachineName}' acquired master role from '{priorMachineName}', Status: '{masterAcquisitonResult?.PriorMachineStatus}', LastHeartbeat: '{masterAcquisitonResult?.PriorMasterLastHeartbeat}'"); } return Role.Master; } else { return Role.Worker; } } }, Counters[GlobalStoreCounters.UpdateRole])); }
/// <inheritdoc /> public Task <Result <CheckpointState> > GetCheckpointStateAsync(OperationContext context) { return(context.PerformOperationAsync( Tracer, async() => { var(checkpoints, startCursor) = await _checkpointsKey.UseReplicatedHashAsync( context, _configuration.RetryWindow, RedisOperation.GetCheckpoint, (batch, key) => batch.GetCheckpointsInfoAsync(key, _clock.UtcNow)) .ThrowIfFailureAsync(); var roleResult = await UpdateRoleAsync(context, release: false); if (!roleResult.Succeeded) { return new ErrorResult(roleResult).AsResult <Result <CheckpointState> >(); } _role = roleResult.Value; var maxCheckpoint = checkpoints.MaxByOrDefault(c => c.CheckpointCreationTime); if (maxCheckpoint == null) { Tracer.Debug(context, $"Getting checkpoint state: Can't find a checkpoint: Start cursor time: {startCursor}"); // Add slack for start cursor to account for clock skew between event hub and redis var epochStartCursor = startCursor - _configuration.EventStore.NewEpochEventStartCursorDelay; return CheckpointState.CreateUnavailable(_role.Value, epochStartCursor); } Tracer.Debug(context, $"Getting checkpoint state: Found checkpoint '{maxCheckpoint}'"); return Result.Success(new CheckpointState(_role.Value, new EventSequencePoint(maxCheckpoint.SequenceNumber), maxCheckpoint.CheckpointId, maxCheckpoint.CheckpointCreationTime, new MachineLocation(maxCheckpoint.MachineName))); }, Counters[GlobalStoreCounters.GetCheckpointState], // Using a timeout to make sure the operation finishes: this is important because we want for heartbeat operations // that call this method to keep running to avoid stale checkpoints. timeout: Configuration.GetCheckpointStateTimeout)); }