private void RefreshJobFromFile() { var retryCount = 3; while (true) { try { var currentFileHash = GetChecksum(_legacyCliModel.PublishedNodesFile); if (currentFileHash != _lastKnownFileHash) { _logger.Information("File {publishedNodesFile} has changed, reloading...", _legacyCliModel.PublishedNodesFile); _lastKnownFileHash = currentFileHash; using (var reader = new StreamReader(_legacyCliModel.PublishedNodesFile)) { var jobs = _publishedNodesJobConverter.Read(reader, _legacyCliModel); var flattened = Flatten(jobs); var serializedJob = _jobSerializer.SerializeJobConfiguration(flattened, out var jobConfigurationType); _jobProcessingInstructionModel = new JobProcessingInstructionModel { Job = new JobInfoModel { Demands = new List <DemandModel>(), Id = "LegacyJob" + "_" + _identity.DeviceId + "_" + _identity.ModuleId, JobConfiguration = serializedJob, JobConfigurationType = jobConfigurationType, LifetimeData = new JobLifetimeDataModel(), Name = "LegacyJob" + "_" + _identity.DeviceId + "_" + _identity.ModuleId, RedundancyConfig = new RedundancyConfigModel { DesiredActiveAgents = 1, DesiredPassiveAgents = 0 } }, ProcessMode = ProcessMode.Active }; _updated = true; } } break; } catch (IOException ex) { retryCount--; if (retryCount > 0) { _logger.Error("Error while loading job from file, retrying..."); } else { _logger.Error(ex, "Error while loading job from file. Retry expired, giving up."); break; } } } }
/// <summary> /// Process jobs /// </summary> private async Task ProcessAsync(JobProcessingInstructionModel jobProcessInstruction, CancellationToken ct) { var currentProcessInstruction = jobProcessInstruction ?? throw new ArgumentNullException(nameof(jobProcessInstruction)); try { // Stop worker heartbeat to start the job heartbeat process _heartbeatTimer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); // Stop worker heartbeat _logger.Information("Worker {WorkerId} processing job {JobId}, mode: {ProcessMode}", WorkerId, currentProcessInstruction.Job.Id, currentProcessInstruction.ProcessMode); // Execute processor while (true) { ct.ThrowIfCancellationRequested(); if (_jobProcess == null) { _jobProcess = new JobProcess(this, currentProcessInstruction, _lifetimeScope, _logger); } else { _jobProcess.ProcessNewInstruction(currentProcessInstruction); } await _jobProcess.WaitAsync(ct).ConfigureAwait(false); // Does not throw // Check if the job is to be continued with new configuration settings if (_jobProcess?.JobContinuation?.Job?.JobConfiguration == null || _jobProcess?.JobContinuation?.ProcessMode == null) { await StopJobProcess().ConfigureAwait(false); break; } currentProcessInstruction = _jobProcess.JobContinuation; _logger.Information("Worker {WorkerId} processing job {JobId} continuation in mode {ProcessMode}", WorkerId, currentProcessInstruction.Job.Id, currentProcessInstruction.ProcessMode); } } catch (OperationCanceledException) { _logger.Information("Worker {WorkerId} cancellation received ...", WorkerId); await StopJobProcess().ConfigureAwait(false); } finally { _logger.Information("Worker: {WorkerId}, Job: {JobId} processing completed ... ", WorkerId, currentProcessInstruction.Job.Id); if (!ct.IsCancellationRequested) { _heartbeatTimer.Change(TimeSpan.Zero, Timeout.InfiniteTimeSpan); // restart worker heartbeat } } }
/// <summary> /// Create model /// </summary> /// <param name="model"></param> public static JobProcessingInstructionApiModel ToApiModel( this JobProcessingInstructionModel model) { if (model == null) { return(null); } return(new JobProcessingInstructionApiModel { ProcessMode = (Api.Jobs.Models.ProcessMode?)model.ProcessMode, Job = model.Job.ToApiModel() }); }
/// <summary> /// Reconfigure the existing process /// </summary> public void ProcessNewInstruction(JobProcessingInstructionModel jobProcessInstruction) { _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource( _outer._cts.Token); _currentJobProcessInstruction = jobProcessInstruction; var jobConfig = _outer._jobConfigurationFactory.DeserializeJobConfiguration( Job.JobConfiguration, Job.JobConfigurationType); _currentProcessingEngine.ReconfigureTrigger(jobConfig); JobContinuation = null; _processor = Task.Run(() => ProcessAsync()); }
/// <summary> /// Receives the heartbeat from the LegacyJobOrchestrator, JobProcess; used to control lifetime of job (cancel, restart, keep). /// </summary> /// <param name="heartbeat"></param> /// <param name="ct"></param> /// <returns></returns> public Task <HeartbeatResultModel> SendHeartbeatAsync(HeartbeatModel heartbeat, CancellationToken ct = default) { _lock.Wait(ct); try { HeartbeatResultModel heartbeatResultModel; JobProcessingInstructionModel job = null; if (heartbeat.Job != null) { if (_assignedJobs.TryGetValue(heartbeat.Worker.WorkerId, out job) && job.Job.Id == heartbeat.Job.JobId) { // JobProcess should keep working heartbeatResultModel = new HeartbeatResultModel { HeartbeatInstruction = HeartbeatInstruction.Keep, LastActiveHeartbeat = DateTime.UtcNow, UpdatedJob = null, }; } else { // JobProcess have to finished current and process new job (if job != null) otherwise complete heartbeatResultModel = new HeartbeatResultModel { HeartbeatInstruction = HeartbeatInstruction.CancelProcessing, LastActiveHeartbeat = DateTime.UtcNow, UpdatedJob = job, }; } } else { // usecase when called from Timer of Worker instead of JobProcess heartbeatResultModel = new HeartbeatResultModel { HeartbeatInstruction = HeartbeatInstruction.Keep, LastActiveHeartbeat = DateTime.UtcNow, UpdatedJob = null, }; } _logger.Debug("SendHeartbeatAsync updated worker {worker} with {heartbeatInstruction} instruction for job {jobId}.", heartbeat.Worker.WorkerId, heartbeatResultModel?.HeartbeatInstruction, job?.Job?.Id); return(Task.FromResult(heartbeatResultModel)); } catch (OperationCanceledException) { _logger.Information("Operation SendHeartbeatAsync was canceled"); throw; } finally { _lock.Release(); } }
/// <summary> /// Process jobs /// </summary> /// <returns></returns> private async Task ProcessAsync(JobProcessingInstructionModel jobProcessInstruction, CancellationToken ct) { try { // Stop worker heartbeat to start the job heartbeat process _heartbeatTimer.Change(-1, -1); // Stop worker heartbeat _logger.Information("Worker: {WorkerId} processing job: {JobId}, mode: {ProcessMode}", WorkerId, jobProcessInstruction.Job.Id, jobProcessInstruction.ProcessMode); // Execute processor while (true) { _jobProcess = null; ct.ThrowIfCancellationRequested(); using (_jobProcess = new JobProcess(this, jobProcessInstruction, _lifetimeScope, _logger)) { await _jobProcess.WaitAsync(ct).ConfigureAwait(false); // Does not throw } // Check if the job is to be continued with new configuration settings if (_jobProcess.JobContinuation == null) { _jobProcess = null; break; } jobProcessInstruction = _jobProcess.JobContinuation; if (jobProcessInstruction?.Job?.JobConfiguration == null || jobProcessInstruction?.ProcessMode == null) { _logger.Information("Job continuation invalid, continue listening..."); _jobProcess = null; break; } _logger.Information("Processing job continuation..."); } } catch (OperationCanceledException) { _logger.Information("Processing cancellation received ..."); _jobProcess = null; } finally { _logger.Information("Worker: {WorkerId}, Job: {JobId} processing completed ... ", WorkerId, jobProcessInstruction.Job.Id); if (!ct.IsCancellationRequested) { _heartbeatTimer.Change(0, -1); // restart worker heartbeat } } }
/// <summary> /// Process jobs /// </summary> /// <returns></returns> private async Task ProcessAsync(JobProcessingInstructionModel jobProcessInstruction, CancellationToken ct) { try { // Stop worker heartbeat to start the job heartbeat process _heartbeatTimer.Change(-1, -1); // Stop worker heartbeat _logger.Information("Starting to process new job..."); // Execute processor while (true) { _jobProcess = null; ct.ThrowIfCancellationRequested(); using (_jobProcess = new JobProcess(this, jobProcessInstruction, _lifetimeScope, _logger)) { await _jobProcess.WaitAsync(); // Does not throw } // Check if the job is to be continued with new configuration settings if (_jobProcess.JobContinuation == null) { _jobProcess = null; break; } jobProcessInstruction = _jobProcess.JobContinuation; if (jobProcessInstruction?.Job?.JobConfiguration == null || jobProcessInstruction?.ProcessMode == null) { _logger.Information("Job continuation invalid, continue listening..."); _jobProcess = null; break; } _logger.Information("Processing job continuation..."); } } finally { _logger.Information("Job processing completed..."); if (!ct.IsCancellationRequested) { _heartbeatTimer.Change(0, -1); // restart worker heartbeat } } }
/// <summary> /// Create processor /// </summary> /// <param name="outer"></param> /// <param name="jobProcessInstruction"></param> /// <param name="workerScope"></param> /// <param name="logger"></param> public JobProcess(Worker outer, JobProcessingInstructionModel jobProcessInstruction, ILifetimeScope workerScope, ILogger logger) { _outer = outer; _logger = logger.ForContext <JobProcess>(); _currentJobProcessInstruction = jobProcessInstruction; _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource( _outer._cts.Token); // Do autofac injection to resolve processing engine var jobConfig = _outer._jobConfigurationFactory.DeserializeJobConfiguration( Job.JobConfiguration, Job.JobConfigurationType); var jobProcessorFactory = workerScope.ResolveNamed <IProcessingEngineContainerFactory>( Job.JobConfigurationType, new NamedParameter(nameof(jobConfig), jobConfig)); _jobScope = workerScope.BeginLifetimeScope( jobProcessorFactory.GetJobContainerScope(outer.WorkerId, Job.Id)); _currentProcessingEngine = _jobScope.Resolve <IProcessingEngine>(); // Continuously send job status heartbeats _heartbeatTimer = new Timer(_ => OnHeartbeatTimerAsync().Wait()); _processor = Task.Run(() => ProcessAsync()); }