/// <summary>
/// Create document
/// </summary>
/// <param name="model"></param>
/// <param name="jobId"></param>
/// <returns></returns>
public static ProcessingStatusDocument ToDocumentModel(
this ProcessingStatusModel model, string jobId)
{
return(new ProcessingStatusDocument {
LastKnownHeartbeat = model.LastKnownHeartbeat,
LastKnownState = model.LastKnownState,
ProcessMode = model.ProcessMode,
JobId = jobId
});
}
/// <summary>
/// Clone
/// </summary>
/// <param name="model"></param>
/// <returns></returns>
public static ProcessingStatusModel Clone(this ProcessingStatusModel model)
{
if (model == null)
{
return(null);
}
return(new ProcessingStatusModel {
LastKnownHeartbeat = model.LastKnownHeartbeat,
LastKnownState = model.LastKnownState,
ProcessMode = model.ProcessMode
});
}
/// <summary>
/// Create model
/// </summary>
/// <param name="model"></param>
public static ProcessingStatusApiModel ToApiModel(
this ProcessingStatusModel model)
{
if (model == null)
{
return(null);
}
return(new ProcessingStatusApiModel {
LastKnownHeartbeat = model.LastKnownHeartbeat,
LastKnownState = model.LastKnownState?.DeepClone(),
ProcessMode = (Api.Jobs.Models.ProcessMode?)model.ProcessMode
});
}
/// <inheritdoc/>
public async Task <HeartbeatResultModel> SendHeartbeatAsync(HeartbeatModel heartbeat,
CancellationToken ct)
{
if (_workerRepository != null)
{
await _workerRepository.AddOrUpdate(heartbeat.Worker);
}
var result = new HeartbeatResultModel {
HeartbeatInstruction = HeartbeatInstruction.Keep,
LastActiveHeartbeat = null,
UpdatedJob = null
};
if (heartbeat.Job == null)
{
// Worker heartbeat
return(result);
}
var job = await _jobRepository.UpdateAsync(heartbeat.Job.JobId, existingJob => {
if (existingJob.GetHashSafe() != heartbeat.Job.JobHash)
{
// job was updated - instruct worker to reset
result.UpdatedJob = new JobProcessingInstructionModel {
Job = existingJob,
ProcessMode = heartbeat.Job.ProcessMode
};
}
if (existingJob.LifetimeData == null)
{
existingJob.LifetimeData = new JobLifetimeDataModel();
}
if (existingJob.LifetimeData.Status == JobStatus.Canceled ||
existingJob.LifetimeData.Status == JobStatus.Deleted)
{
result.HeartbeatInstruction = HeartbeatInstruction.CancelProcessing;
result.UpdatedJob = null;
result.LastActiveHeartbeat = null;
}
if (result.HeartbeatInstruction == HeartbeatInstruction.Keep)
{
existingJob.LifetimeData.Status = heartbeat.Job.Status;
}
var processingStatus = new ProcessingStatusModel {
LastKnownHeartbeat = DateTime.UtcNow,
LastKnownState = heartbeat.Job.State,
ProcessMode = // Unset processing mode to do correct calculation of active agents
heartbeat.Job.Status == JobStatus.Active ?
heartbeat.Job.ProcessMode : (ProcessMode?)null
};
if (existingJob.LifetimeData.ProcessingStatus == null)
{
existingJob.LifetimeData.ProcessingStatus = new Dictionary <string, ProcessingStatusModel>();
}
existingJob.LifetimeData.ProcessingStatus[heartbeat.Worker.WorkerId] = processingStatus;
var numberOfActiveAgents = existingJob.LifetimeData.ProcessingStatus
.Count(j =>
j.Value.ProcessMode == ProcessMode.Active &&
j.Value.LastKnownHeartbeat > DateTime.UtcNow.Subtract(_jobOrchestratorConfig.JobStaleTime));
if (processingStatus.ProcessMode == ProcessMode.Passive &&
numberOfActiveAgents < existingJob.RedundancyConfig.DesiredActiveAgents)
{
var lastActiveHeartbeat = existingJob.LifetimeData.ProcessingStatus
.Where(s => s.Value.ProcessMode == ProcessMode.Active)
.OrderByDescending(s => s.Value.LastKnownHeartbeat)
.Select(s => s.Value.LastKnownHeartbeat)
.FirstOrDefault();
// Switch this passive agent to active
result.HeartbeatInstruction = HeartbeatInstruction.SwitchToActive;
result.LastActiveHeartbeat = lastActiveHeartbeat;
existingJob.LifetimeData.ProcessingStatus[heartbeat.Worker.WorkerId].ProcessMode =
ProcessMode.Active;
}
return(Task.FromResult(true));
}, ct);
return(result);
}
