/// <summary> /// Receives the heartbeat from the agent. Lifetime information is not persisted in this implementation. This method is /// only used if the /// publishednodes.json file has changed. Is that the case, the worker is informed to cancel (and restart) processing. /// </summary> /// <param name="heartbeat"></param> /// <param name="ct"></param> /// <returns></returns> public Task <HeartbeatResultModel> SendHeartbeatAsync(HeartbeatModel heartbeat, CancellationToken ct = default) { HeartbeatResultModel heartbeatResultModel; if (_updated && heartbeat.Job != null) { if (_availableJobs.Count == 0) { _updated = false; } heartbeatResultModel = new HeartbeatResultModel { HeartbeatInstruction = HeartbeatInstruction.CancelProcessing, LastActiveHeartbeat = DateTime.UtcNow, UpdatedJob = _assignedJobs.TryGetValue(heartbeat.Worker.WorkerId, out var job) ? job : null }; } else { heartbeatResultModel = new HeartbeatResultModel { HeartbeatInstruction = HeartbeatInstruction.Keep, LastActiveHeartbeat = DateTime.UtcNow, UpdatedJob = null }; } return(Task.FromResult(heartbeatResultModel)); }
/// <summary> /// Receives the heartbeat from the LegacyJobOrchestrator, JobProcess; used to control lifetime of job (cancel, restart, keep). /// </summary> /// <param name="heartbeat"></param> /// <param name="ct"></param> /// <returns></returns> public Task <HeartbeatResultModel> SendHeartbeatAsync(HeartbeatModel heartbeat, CancellationToken ct = default) { _lock.Wait(ct); try { HeartbeatResultModel heartbeatResultModel; JobProcessingInstructionModel job = null; if (heartbeat.Job != null) { if (_assignedJobs.TryGetValue(heartbeat.Worker.WorkerId, out job) && job.Job.Id == heartbeat.Job.JobId) { // JobProcess should keep working heartbeatResultModel = new HeartbeatResultModel { HeartbeatInstruction = HeartbeatInstruction.Keep, LastActiveHeartbeat = DateTime.UtcNow, UpdatedJob = null, }; } else { // JobProcess have to finished current and process new job (if job != null) otherwise complete heartbeatResultModel = new HeartbeatResultModel { HeartbeatInstruction = HeartbeatInstruction.CancelProcessing, LastActiveHeartbeat = DateTime.UtcNow, UpdatedJob = job, }; } } else { // usecase when called from Timer of Worker instead of JobProcess heartbeatResultModel = new HeartbeatResultModel { HeartbeatInstruction = HeartbeatInstruction.Keep, LastActiveHeartbeat = DateTime.UtcNow, UpdatedJob = null, }; } _logger.Debug("SendHeartbeatAsync updated worker {worker} with {heartbeatInstruction} instruction for job {jobId}.", heartbeat.Worker.WorkerId, heartbeatResultModel?.HeartbeatInstruction, job?.Job?.Id); return(Task.FromResult(heartbeatResultModel)); } catch (OperationCanceledException) { _logger.Information("Operation SendHeartbeatAsync was canceled"); throw; } finally { _lock.Release(); } }
/// <summary> /// Create response /// </summary> /// <param name="model"></param> /// <returns></returns> public static HeartbeatResponseApiModel ToApiModel( this HeartbeatResultModel model) { if (model == null) { return(null); } return(new HeartbeatResponseApiModel { HeartbeatInstruction = (Api.Jobs.Models.HeartbeatInstruction)model.HeartbeatInstruction, LastActiveHeartbeat = model.LastActiveHeartbeat, UpdatedJob = model.UpdatedJob.ToApiModel() }); }
/// <summary> /// Receives the heartbeat from the agent. Lifetime information is not persisted in this implementation. This method is /// only used if the /// publishednodes.json file has changed. Is that the case, the worker is informed to cancel (and restart) processing. /// </summary> /// <param name="heartbeat"></param> /// <param name="ct"></param> /// <returns></returns> public Task <HeartbeatResultModel> SendHeartbeatAsync(HeartbeatModel heartbeat, CancellationToken ct = default) { HeartbeatResultModel heartbeatResultModel; if (_updated && heartbeat.Job != null) { _updated = false; heartbeatResultModel = new HeartbeatResultModel { HeartbeatInstruction = HeartbeatInstruction.CancelProcessing, LastActiveHeartbeat = DateTime.UtcNow, UpdatedJob = _jobProcessingInstructionModel }; } else { heartbeatResultModel = new HeartbeatResultModel { HeartbeatInstruction = HeartbeatInstruction.Keep, LastActiveHeartbeat = DateTime.UtcNow, UpdatedJob = null }; } return(Task.FromResult(heartbeatResultModel)); }
/// <inheritdoc/> public async Task <HeartbeatResultModel> SendHeartbeatAsync(HeartbeatModel heartbeat, CancellationToken ct) { if (_workerRepository != null) { await _workerRepository.AddOrUpdate(heartbeat.Worker); } var result = new HeartbeatResultModel { HeartbeatInstruction = HeartbeatInstruction.Keep, LastActiveHeartbeat = null, UpdatedJob = null }; if (heartbeat.Job == null) { // Worker heartbeat return(result); } var job = await _jobRepository.UpdateAsync(heartbeat.Job.JobId, existingJob => { if (existingJob.GetHashSafe() != heartbeat.Job.JobHash) { // job was updated - instruct worker to reset result.UpdatedJob = new JobProcessingInstructionModel { Job = existingJob, ProcessMode = heartbeat.Job.ProcessMode }; } if (existingJob.LifetimeData == null) { existingJob.LifetimeData = new JobLifetimeDataModel(); } if (existingJob.LifetimeData.Status == JobStatus.Canceled || existingJob.LifetimeData.Status == JobStatus.Deleted) { result.HeartbeatInstruction = HeartbeatInstruction.CancelProcessing; result.UpdatedJob = null; result.LastActiveHeartbeat = null; } if (result.HeartbeatInstruction == HeartbeatInstruction.Keep) { existingJob.LifetimeData.Status = heartbeat.Job.Status; } var processingStatus = new ProcessingStatusModel { LastKnownHeartbeat = DateTime.UtcNow, LastKnownState = heartbeat.Job.State, ProcessMode = // Unset processing mode to do correct calculation of active agents heartbeat.Job.Status == JobStatus.Active ? heartbeat.Job.ProcessMode : (ProcessMode?)null }; if (existingJob.LifetimeData.ProcessingStatus == null) { existingJob.LifetimeData.ProcessingStatus = new Dictionary <string, ProcessingStatusModel>(); } existingJob.LifetimeData.ProcessingStatus[heartbeat.Worker.WorkerId] = processingStatus; var numberOfActiveAgents = existingJob.LifetimeData.ProcessingStatus .Count(j => j.Value.ProcessMode == ProcessMode.Active && j.Value.LastKnownHeartbeat > DateTime.UtcNow.Subtract(_jobOrchestratorConfig.JobStaleTime)); if (processingStatus.ProcessMode == ProcessMode.Passive && numberOfActiveAgents < existingJob.RedundancyConfig.DesiredActiveAgents) { var lastActiveHeartbeat = existingJob.LifetimeData.ProcessingStatus .Where(s => s.Value.ProcessMode == ProcessMode.Active) .OrderByDescending(s => s.Value.LastKnownHeartbeat) .Select(s => s.Value.LastKnownHeartbeat) .FirstOrDefault(); // Switch this passive agent to active result.HeartbeatInstruction = HeartbeatInstruction.SwitchToActive; result.LastActiveHeartbeat = lastActiveHeartbeat; existingJob.LifetimeData.ProcessingStatus[heartbeat.Worker.WorkerId].ProcessMode = ProcessMode.Active; } return(Task.FromResult(true)); }, ct); return(result); }