/// <summary>
/// Receives the heartbeat from the agent. Lifetime information is not persisted in this implementation. This method is
/// only used if the
/// publishednodes.json file has changed. Is that the case, the worker is informed to cancel (and restart) processing.
/// </summary>
/// <param name="heartbeat"></param>
/// <param name="ct"></param>
/// <returns></returns>
public Task <HeartbeatResultModel> SendHeartbeatAsync(HeartbeatModel heartbeat, CancellationToken ct = default)
{
HeartbeatResultModel heartbeatResultModel;
if (_updated && heartbeat.Job != null)
{
if (_availableJobs.Count == 0)
{
_updated = false;
}
heartbeatResultModel = new HeartbeatResultModel {
HeartbeatInstruction = HeartbeatInstruction.CancelProcessing,
LastActiveHeartbeat = DateTime.UtcNow,
UpdatedJob = _assignedJobs.TryGetValue(heartbeat.Worker.WorkerId, out var job) ? job : null
};
}
else
{
heartbeatResultModel = new HeartbeatResultModel {
HeartbeatInstruction = HeartbeatInstruction.Keep,
LastActiveHeartbeat = DateTime.UtcNow,
UpdatedJob = null
};
}
return(Task.FromResult(heartbeatResultModel));
}
/// <summary>
/// Receives the heartbeat from the LegacyJobOrchestrator, JobProcess; used to control lifetime of job (cancel, restart, keep).
/// </summary>
/// <param name="heartbeat"></param>
/// <param name="ct"></param>
/// <returns></returns>
public Task <HeartbeatResultModel> SendHeartbeatAsync(HeartbeatModel heartbeat, CancellationToken ct = default)
{
_lock.Wait(ct);
try {
HeartbeatResultModel heartbeatResultModel;
JobProcessingInstructionModel job = null;
if (heartbeat.Job != null)
{
if (_assignedJobs.TryGetValue(heartbeat.Worker.WorkerId, out job) &&
job.Job.Id == heartbeat.Job.JobId)
{
// JobProcess should keep working
heartbeatResultModel = new HeartbeatResultModel {
HeartbeatInstruction = HeartbeatInstruction.Keep,
LastActiveHeartbeat = DateTime.UtcNow,
UpdatedJob = null,
};
}
else
{
// JobProcess have to finished current and process new job (if job != null) otherwise complete
heartbeatResultModel = new HeartbeatResultModel {
HeartbeatInstruction = HeartbeatInstruction.CancelProcessing,
LastActiveHeartbeat = DateTime.UtcNow,
UpdatedJob = job,
};
}
}
else
{
// usecase when called from Timer of Worker instead of JobProcess
heartbeatResultModel = new HeartbeatResultModel {
HeartbeatInstruction = HeartbeatInstruction.Keep,
LastActiveHeartbeat = DateTime.UtcNow,
UpdatedJob = null,
};
}
_logger.Debug("SendHeartbeatAsync updated worker {worker} with {heartbeatInstruction} instruction for job {jobId}.",
heartbeat.Worker.WorkerId,
heartbeatResultModel?.HeartbeatInstruction,
job?.Job?.Id);
return(Task.FromResult(heartbeatResultModel));
}
catch (OperationCanceledException) {
_logger.Information("Operation SendHeartbeatAsync was canceled");
throw;
}
finally {
_lock.Release();
}
}
/// <summary>
/// Create response
/// </summary>
/// <param name="model"></param>
/// <returns></returns>
public static HeartbeatResponseApiModel ToApiModel(
this HeartbeatResultModel model)
{
if (model == null)
{
return(null);
}
return(new HeartbeatResponseApiModel {
HeartbeatInstruction = (Api.Jobs.Models.HeartbeatInstruction)model.HeartbeatInstruction,
LastActiveHeartbeat = model.LastActiveHeartbeat,
UpdatedJob = model.UpdatedJob.ToApiModel()
});
}
/// <summary>
/// Receives the heartbeat from the agent. Lifetime information is not persisted in this implementation. This method is
/// only used if the
/// publishednodes.json file has changed. Is that the case, the worker is informed to cancel (and restart) processing.
/// </summary>
/// <param name="heartbeat"></param>
/// <param name="ct"></param>
/// <returns></returns>
public Task <HeartbeatResultModel> SendHeartbeatAsync(HeartbeatModel heartbeat, CancellationToken ct = default)
{
HeartbeatResultModel heartbeatResultModel;
if (_updated && heartbeat.Job != null)
{
_updated = false;
heartbeatResultModel = new HeartbeatResultModel {
HeartbeatInstruction = HeartbeatInstruction.CancelProcessing, LastActiveHeartbeat = DateTime.UtcNow, UpdatedJob = _jobProcessingInstructionModel
};
}
else
{
heartbeatResultModel = new HeartbeatResultModel {
HeartbeatInstruction = HeartbeatInstruction.Keep, LastActiveHeartbeat = DateTime.UtcNow, UpdatedJob = null
};
}
return(Task.FromResult(heartbeatResultModel));
}
/// <inheritdoc/>
public async Task <HeartbeatResultModel> SendHeartbeatAsync(HeartbeatModel heartbeat,
CancellationToken ct)
{
if (_workerRepository != null)
{
await _workerRepository.AddOrUpdate(heartbeat.Worker);
}
var result = new HeartbeatResultModel {
HeartbeatInstruction = HeartbeatInstruction.Keep,
LastActiveHeartbeat = null,
UpdatedJob = null
};
if (heartbeat.Job == null)
{
// Worker heartbeat
return(result);
}
var job = await _jobRepository.UpdateAsync(heartbeat.Job.JobId, existingJob => {
if (existingJob.GetHashSafe() != heartbeat.Job.JobHash)
{
// job was updated - instruct worker to reset
result.UpdatedJob = new JobProcessingInstructionModel {
Job = existingJob,
ProcessMode = heartbeat.Job.ProcessMode
};
}
if (existingJob.LifetimeData == null)
{
existingJob.LifetimeData = new JobLifetimeDataModel();
}
if (existingJob.LifetimeData.Status == JobStatus.Canceled ||
existingJob.LifetimeData.Status == JobStatus.Deleted)
{
result.HeartbeatInstruction = HeartbeatInstruction.CancelProcessing;
result.UpdatedJob = null;
result.LastActiveHeartbeat = null;
}
if (result.HeartbeatInstruction == HeartbeatInstruction.Keep)
{
existingJob.LifetimeData.Status = heartbeat.Job.Status;
}
var processingStatus = new ProcessingStatusModel {
LastKnownHeartbeat = DateTime.UtcNow,
LastKnownState = heartbeat.Job.State,
ProcessMode = // Unset processing mode to do correct calculation of active agents
heartbeat.Job.Status == JobStatus.Active ?
heartbeat.Job.ProcessMode : (ProcessMode?)null
};
if (existingJob.LifetimeData.ProcessingStatus == null)
{
existingJob.LifetimeData.ProcessingStatus = new Dictionary <string, ProcessingStatusModel>();
}
existingJob.LifetimeData.ProcessingStatus[heartbeat.Worker.WorkerId] = processingStatus;
var numberOfActiveAgents = existingJob.LifetimeData.ProcessingStatus
.Count(j =>
j.Value.ProcessMode == ProcessMode.Active &&
j.Value.LastKnownHeartbeat > DateTime.UtcNow.Subtract(_jobOrchestratorConfig.JobStaleTime));
if (processingStatus.ProcessMode == ProcessMode.Passive &&
numberOfActiveAgents < existingJob.RedundancyConfig.DesiredActiveAgents)
{
var lastActiveHeartbeat = existingJob.LifetimeData.ProcessingStatus
.Where(s => s.Value.ProcessMode == ProcessMode.Active)
.OrderByDescending(s => s.Value.LastKnownHeartbeat)
.Select(s => s.Value.LastKnownHeartbeat)
.FirstOrDefault();
// Switch this passive agent to active
result.HeartbeatInstruction = HeartbeatInstruction.SwitchToActive;
result.LastActiveHeartbeat = lastActiveHeartbeat;
existingJob.LifetimeData.ProcessingStatus[heartbeat.Worker.WorkerId].ProcessMode =
ProcessMode.Active;
}
return(Task.FromResult(true));
}, ct);
return(result);
}