private void TrySendProgressServiceBusMessage(Geres.Common.Entities.Job job, string progress) { try { _notificationServiceBusClient.SendProgressMessage(job, progress); } catch (Exception ex) { Geres.Diagnostics.GeresEventSource.Log.EngineJobHostJobProcessorFailedSendingServiceBusMessage( RoleEnvironment.CurrentRoleInstance.Id, RoleEnvironment.DeploymentId, GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_JOBSTATUS, GlobalConstants.SERVICEBUS_INTERNAL_SUBSCRIPTION_JOBPROGRESS, ex.Message, ex.ToString() ); } }
private void TrySendServiceBusFinishedMessage(JobStatus jobStatus, JobProcessResult jobProcessResult, Geres.Common.Entities.Job job) { try { _notificationServiceBusClient.SendFinishedMessage(job, jobStatus, jobProcessResult.Output); } catch (Exception ex) { Geres.Diagnostics.GeresEventSource.Log.EngineJobHostJobProcessorFailedSendingServiceBusMessage( RoleEnvironment.CurrentRoleInstance.Id, RoleEnvironment.DeploymentId, GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_JOBSTATUS, GlobalConstants.SERVICEBUS_INTERNAL_SUBSCRIPTION_JOBFINISHED, ex.Message, ex.ToString() ); } }
public bool Process(string dedicatedBatchId) { CloudQueue queue = null; OperationContext getMessageContext = new OperationContext(); OperationContext deleteMessageContext = new OperationContext(); var jobStatus = JobStatus.Submitted; var jobOutput = string.Empty; JobProcessResult jobProcessResult = new JobProcessResult(); // // Get the list of batches (equivalet to queues) from table storage and order them based on their priority // var batches = GetPrioritizedListOfBatches(dedicatedBatchId); // // Find a message in the list of batches and process the first message found // BatchEntity singleBatch = null; CloudQueueMessage message = null; foreach (var batch in batches) { // Create the queue client. queue = _queueClient.GetQueueReference(batch.Id); // // Try getting a message from the queue // try { message = TryDequeueMessage(getMessageContext, batch.Id, queue); if (message != null) { singleBatch = batch; GeresEventSource.Log.EngineJobHostReceivedMessage(message.AsString); break; } } catch (Exception ex) { message = null; GeresEventSource.Log.EngineJobHostFailedDequeueMessage(batch.Id, batch.Name, ex.Message, ex.StackTrace); break; } } // Stop processing if there is no message if (message == null) { GeresEventSource.Log.EngineJobHostNoMessageAvailable(); return(false); } // // If there is a message available for processing, try lookup the job for it // Geres.Common.Entities.Job job = GetJob(message.AsString, singleBatch.Id); if (job == null) { GeresEventSource.Log.EngineJobHostJobForMessageNotFound(message.AsString, singleBatch.Id); // Nevertheless return true since the Processor dequeued a message return(true); } else if ((job.Status == JobStatus.InProgress) || (job.Status == JobStatus.Started)) { // The job is running, already, on another worker ... skip the job from being executed again Geres.Diagnostics.GeresEventSource.Log.EngineJobHostSkipRunningJobSinceItIsStartedAlready(job.JobId, job.JobType, singleBatch.Id); // Return true since we dequeued a message from the queue return(true); } else { // Job found, send started message since everything below counts to job processing, already TrySendStartServiceBusMessage(job); } // // Try deploying the tenant and the code to process the job to an area on the machine // If the path is null then the tenant failed to deploy // var tenantJobContext = TryDeployTenant(job, singleBatch.Id); if (tenantJobContext == null) { return(true); } // // Looking up the built-in job processor. There should always be a built-in job processor // since the last fall-back will be the job processor worker process // GeresEventSource.Log.EngineJobHostLookingUpProcessor(string.Format("built-in job processor selection for {0}", job.JobType)); IJobImplementation processor = _builtInJobsFactory.Lookup(job); if (processor == null) { GeresEventSource.Log.EngineJobHostProcessorLookupFailed("no built-in job processor returned - BUG in system!!"); return(true); } else { // Initialize the built-in job if it supports initialization if (processor is IJobBuiltInImplementation) { ((IJobBuiltInImplementation)processor).InitializeContextBeforeExecution ( new BuiltInJobInitializationContext() { ExecutionAsUserName = tenantJobContext.UserName, ExecutionAsUserPassword = tenantJobContext.UserPassword } ); } } // // Job Processor created successfully, hence continue processing // Setup a service-bus subscription to allow client-side cancellation // SubscriptionClient cancellationSubscriptionClient = null; string cancellationSubscriptionName = Guid.NewGuid().ToString(); if (_singleJobCancellationEnabled) { try { cancellationSubscriptionClient = _cancellationServiceBusClient.CreateSubscription(job.JobId, cancellationSubscriptionName); cancellationSubscriptionClient.OnMessage((receivedMessage) => { processor.CancelProcessCallback(); }); } catch (Exception ex) { Geres.Diagnostics.GeresEventSource.Log.EngineJobHostFailedSettingUpCancellationSubscriptionForJob(job.JobId, job.JobType, singleBatch.Id, ex.Message, ex.ToString()); // Set the job to aborted in the job log try { SetJobMonitoringStatus ( job.JobId, singleBatch.Id, JobStatus.AbortedInternalError, string.Empty, true ); } catch (Exception exi) { Geres.Diagnostics.GeresEventSource.Log.EngineJobHostFailedUpdatingJobStatus(job.JobId, job.JobName, job.JobType, exi.Message, exi.ToString()); } // Job has been dequeued, so return true return(true); } } // // Now try processing the job // try { // Update the status for the job SetJobMonitoringStatus(job.JobId, singleBatch.Id, JobStatus.Started, string.Empty, true); // simple flag so that the job status table is not constantly updated, i.e. update it once. var updateProgress = true; GeresEventSource.Log.EngineJobHostStartingJobProcessor(job.JobId, job.JobType, singleBatch.Id); // use the newly discovered processor implementation to do the actual work. // the callback provides the 3rd party code with the ability to provide progress updates back to the client jobProcessResult = processor.DoWork(job, tenantJobContext.JobRootPath, tenantJobContext.JobWorkingRootPath, (unit) => { TrySendProgressServiceBusMessage(job, unit); if (updateProgress) { SetJobMonitoringStatus(job.JobId, singleBatch.Id, JobStatus.InProgress, string.Empty, true); updateProgress = false; } }); // Job processing completed, update the status jobStatus = jobProcessResult.Status; jobOutput = jobProcessResult.Output; GeresEventSource.Log.EngineJobHostJobProcessorImplementationCompletedWithStatus(job.JobId, job.JobType, jobStatus.ToString()); } catch (Exception ex) { // Job Processing failed jobStatus = JobStatus.Failed; jobOutput = string.Format("Job did run into uncaught exception: {0}!", ex.Message); GeresEventSource.Log.EngineJobHostJobProcessorImplementationFailed(job.JobId, job.JobType, singleBatch.Id, ex.Message, ex.StackTrace); } // // notify the originator of the job that processing has finished // TrySendServiceBusFinishedMessage(jobStatus, jobProcessResult, job); // update the status of the job try { SetJobMonitoringStatus(job.JobId, singleBatch.Id, jobStatus, jobOutput, false); } catch (Exception ex) { GeresEventSource.Log.EngineJobHostFailedUpdatingJobStatus(job.JobId, job.JobName, job.JobType, ex.Message, ex.StackTrace); } // delete the cancellation subscription for this jobId if (_singleJobCancellationEnabled) { TryDeleteCacellationServiceBusSubscription(cancellationSubscriptionName, job.JobId, job.JobType); } // remove the job directory _tenantManager.DeleteJobDirectory(job, singleBatch.Id); // If the job has any other status but failed, delete the message // If the job status is failed and it got dequeued too often, also delete the message if (jobStatus != JobStatus.Failed) { TryDeleteMessage(singleBatch.Id, queue, message); } else if (message.DequeueCount >= _messageMaxRetryAttempts) { TryDeleteMessage(singleBatch.Id, queue, message); } return(true); }