private void RefreshJobFromFile()
{
var retryCount = 3;
while (true)
{
try {
var currentFileHash = GetChecksum(_legacyCliModel.PublishedNodesFile);
if (currentFileHash != _lastKnownFileHash)
{
_logger.Information("File {publishedNodesFile} has changed, reloading...", _legacyCliModel.PublishedNodesFile);
_lastKnownFileHash = currentFileHash;
using (var reader = new StreamReader(_legacyCliModel.PublishedNodesFile)) {
var jobs = _publishedNodesJobConverter.Read(reader, _legacyCliModel);
var flattened = Flatten(jobs);
var serializedJob = _jobSerializer.SerializeJobConfiguration(flattened, out var jobConfigurationType);
_jobProcessingInstructionModel = new JobProcessingInstructionModel {
Job = new JobInfoModel {
Demands = new List <DemandModel>(),
Id = "LegacyJob" + "_" + _identity.DeviceId + "_" + _identity.ModuleId,
JobConfiguration = serializedJob,
JobConfigurationType = jobConfigurationType,
LifetimeData = new JobLifetimeDataModel(),
Name = "LegacyJob" + "_" + _identity.DeviceId + "_" + _identity.ModuleId,
RedundancyConfig = new RedundancyConfigModel {
DesiredActiveAgents = 1, DesiredPassiveAgents = 0
}
},
ProcessMode = ProcessMode.Active
};
_updated = true;
}
}
break;
}
catch (IOException ex) {
retryCount--;
if (retryCount > 0)
{
_logger.Error("Error while loading job from file, retrying...");
}
else
{
_logger.Error(ex, "Error while loading job from file. Retry expired, giving up.");
break;
}
}
}
}
/// <summary>
/// Process jobs
/// </summary>
private async Task ProcessAsync(JobProcessingInstructionModel jobProcessInstruction,
CancellationToken ct)
{
var currentProcessInstruction = jobProcessInstruction ??
throw new ArgumentNullException(nameof(jobProcessInstruction));
try {
// Stop worker heartbeat to start the job heartbeat process
_heartbeatTimer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); // Stop worker heartbeat
_logger.Information("Worker {WorkerId} processing job {JobId}, mode: {ProcessMode}",
WorkerId, currentProcessInstruction.Job.Id, currentProcessInstruction.ProcessMode);
// Execute processor
while (true)
{
ct.ThrowIfCancellationRequested();
if (_jobProcess == null)
{
_jobProcess = new JobProcess(this, currentProcessInstruction, _lifetimeScope, _logger);
}
else
{
_jobProcess.ProcessNewInstruction(currentProcessInstruction);
}
await _jobProcess.WaitAsync(ct).ConfigureAwait(false); // Does not throw
// Check if the job is to be continued with new configuration settings
if (_jobProcess?.JobContinuation?.Job?.JobConfiguration == null ||
_jobProcess?.JobContinuation?.ProcessMode == null)
{
await StopJobProcess().ConfigureAwait(false);
break;
}
currentProcessInstruction = _jobProcess.JobContinuation;
_logger.Information("Worker {WorkerId} processing job {JobId} continuation in mode {ProcessMode}",
WorkerId, currentProcessInstruction.Job.Id, currentProcessInstruction.ProcessMode);
}
}
catch (OperationCanceledException) {
_logger.Information("Worker {WorkerId} cancellation received ...", WorkerId);
await StopJobProcess().ConfigureAwait(false);
}
finally {
_logger.Information("Worker: {WorkerId}, Job: {JobId} processing completed ... ",
WorkerId, currentProcessInstruction.Job.Id);
if (!ct.IsCancellationRequested)
{
_heartbeatTimer.Change(TimeSpan.Zero, Timeout.InfiniteTimeSpan); // restart worker heartbeat
}
}
}
/// <summary>
/// Create model
/// </summary>
/// <param name="model"></param>
public static JobProcessingInstructionApiModel ToApiModel(
this JobProcessingInstructionModel model)
{
if (model == null)
{
return(null);
}
return(new JobProcessingInstructionApiModel {
ProcessMode = (Api.Jobs.Models.ProcessMode?)model.ProcessMode,
Job = model.Job.ToApiModel()
});
}
/// <summary>
/// Reconfigure the existing process
/// </summary>
public void ProcessNewInstruction(JobProcessingInstructionModel jobProcessInstruction)
{
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(
_outer._cts.Token);
_currentJobProcessInstruction = jobProcessInstruction;
var jobConfig = _outer._jobConfigurationFactory.DeserializeJobConfiguration(
Job.JobConfiguration, Job.JobConfigurationType);
_currentProcessingEngine.ReconfigureTrigger(jobConfig);
JobContinuation = null;
_processor = Task.Run(() => ProcessAsync());
}
/// <summary>
/// Receives the heartbeat from the LegacyJobOrchestrator, JobProcess; used to control lifetime of job (cancel, restart, keep).
/// </summary>
/// <param name="heartbeat"></param>
/// <param name="ct"></param>
/// <returns></returns>
public Task <HeartbeatResultModel> SendHeartbeatAsync(HeartbeatModel heartbeat, CancellationToken ct = default)
{
_lock.Wait(ct);
try {
HeartbeatResultModel heartbeatResultModel;
JobProcessingInstructionModel job = null;
if (heartbeat.Job != null)
{
if (_assignedJobs.TryGetValue(heartbeat.Worker.WorkerId, out job) &&
job.Job.Id == heartbeat.Job.JobId)
{
// JobProcess should keep working
heartbeatResultModel = new HeartbeatResultModel {
HeartbeatInstruction = HeartbeatInstruction.Keep,
LastActiveHeartbeat = DateTime.UtcNow,
UpdatedJob = null,
};
}
else
{
// JobProcess have to finished current and process new job (if job != null) otherwise complete
heartbeatResultModel = new HeartbeatResultModel {
HeartbeatInstruction = HeartbeatInstruction.CancelProcessing,
LastActiveHeartbeat = DateTime.UtcNow,
UpdatedJob = job,
};
}
}
else
{
// usecase when called from Timer of Worker instead of JobProcess
heartbeatResultModel = new HeartbeatResultModel {
HeartbeatInstruction = HeartbeatInstruction.Keep,
LastActiveHeartbeat = DateTime.UtcNow,
UpdatedJob = null,
};
}
_logger.Debug("SendHeartbeatAsync updated worker {worker} with {heartbeatInstruction} instruction for job {jobId}.",
heartbeat.Worker.WorkerId,
heartbeatResultModel?.HeartbeatInstruction,
job?.Job?.Id);
return(Task.FromResult(heartbeatResultModel));
}
catch (OperationCanceledException) {
_logger.Information("Operation SendHeartbeatAsync was canceled");
throw;
}
finally {
_lock.Release();
}
}
/// <summary>
/// Process jobs
/// </summary>
/// <returns></returns>
private async Task ProcessAsync(JobProcessingInstructionModel jobProcessInstruction,
CancellationToken ct)
{
try {
// Stop worker heartbeat to start the job heartbeat process
_heartbeatTimer.Change(-1, -1); // Stop worker heartbeat
_logger.Information("Worker: {WorkerId} processing job: {JobId}, mode: {ProcessMode}",
WorkerId, jobProcessInstruction.Job.Id, jobProcessInstruction.ProcessMode);
// Execute processor
while (true)
{
_jobProcess = null;
ct.ThrowIfCancellationRequested();
using (_jobProcess = new JobProcess(this, jobProcessInstruction,
_lifetimeScope, _logger)) {
await _jobProcess.WaitAsync(ct).ConfigureAwait(false); // Does not throw
}
// Check if the job is to be continued with new configuration settings
if (_jobProcess.JobContinuation == null)
{
_jobProcess = null;
break;
}
jobProcessInstruction = _jobProcess.JobContinuation;
if (jobProcessInstruction?.Job?.JobConfiguration == null ||
jobProcessInstruction?.ProcessMode == null)
{
_logger.Information("Job continuation invalid, continue listening...");
_jobProcess = null;
break;
}
_logger.Information("Processing job continuation...");
}
}
catch (OperationCanceledException) {
_logger.Information("Processing cancellation received ...");
_jobProcess = null;
}
finally {
_logger.Information("Worker: {WorkerId}, Job: {JobId} processing completed ... ",
WorkerId, jobProcessInstruction.Job.Id);
if (!ct.IsCancellationRequested)
{
_heartbeatTimer.Change(0, -1); // restart worker heartbeat
}
}
}
/// <summary>
/// Process jobs
/// </summary>
/// <returns></returns>
private async Task ProcessAsync(JobProcessingInstructionModel jobProcessInstruction,
CancellationToken ct)
{
try {
// Stop worker heartbeat to start the job heartbeat process
_heartbeatTimer.Change(-1, -1); // Stop worker heartbeat
_logger.Information("Starting to process new job...");
// Execute processor
while (true)
{
_jobProcess = null;
ct.ThrowIfCancellationRequested();
using (_jobProcess = new JobProcess(this, jobProcessInstruction,
_lifetimeScope, _logger)) {
await _jobProcess.WaitAsync(); // Does not throw
}
// Check if the job is to be continued with new configuration settings
if (_jobProcess.JobContinuation == null)
{
_jobProcess = null;
break;
}
jobProcessInstruction = _jobProcess.JobContinuation;
if (jobProcessInstruction?.Job?.JobConfiguration == null ||
jobProcessInstruction?.ProcessMode == null)
{
_logger.Information("Job continuation invalid, continue listening...");
_jobProcess = null;
break;
}
_logger.Information("Processing job continuation...");
}
}
finally {
_logger.Information("Job processing completed...");
if (!ct.IsCancellationRequested)
{
_heartbeatTimer.Change(0, -1); // restart worker heartbeat
}
}
}
/// <summary>
/// Create processor
/// </summary>
/// <param name="outer"></param>
/// <param name="jobProcessInstruction"></param>
/// <param name="workerScope"></param>
/// <param name="logger"></param>
public JobProcess(Worker outer, JobProcessingInstructionModel jobProcessInstruction,
ILifetimeScope workerScope, ILogger logger)
{
_outer = outer;
_logger = logger.ForContext <JobProcess>();
_currentJobProcessInstruction = jobProcessInstruction;
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(
_outer._cts.Token);
// Do autofac injection to resolve processing engine
var jobConfig = _outer._jobConfigurationFactory.DeserializeJobConfiguration(
Job.JobConfiguration, Job.JobConfigurationType);
var jobProcessorFactory = workerScope.ResolveNamed <IProcessingEngineContainerFactory>(
Job.JobConfigurationType, new NamedParameter(nameof(jobConfig), jobConfig));
_jobScope = workerScope.BeginLifetimeScope(
jobProcessorFactory.GetJobContainerScope(outer.WorkerId, Job.Id));
_currentProcessingEngine = _jobScope.Resolve <IProcessingEngine>();
// Continuously send job status heartbeats
_heartbeatTimer = new Timer(_ => OnHeartbeatTimerAsync().Wait());
_processor = Task.Run(() => ProcessAsync());
}