private void TrySendProgressServiceBusMessage(Geres.Common.Entities.Job job, string progress)
{
try
{
_notificationServiceBusClient.SendProgressMessage(job, progress);
}
catch (Exception ex)
{
Geres.Diagnostics.GeresEventSource.Log.EngineJobHostJobProcessorFailedSendingServiceBusMessage(
RoleEnvironment.CurrentRoleInstance.Id,
RoleEnvironment.DeploymentId,
GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_JOBSTATUS,
GlobalConstants.SERVICEBUS_INTERNAL_SUBSCRIPTION_JOBPROGRESS,
ex.Message,
ex.ToString()
);
}
}
private void TrySendServiceBusFinishedMessage(JobStatus jobStatus, JobProcessResult jobProcessResult, Geres.Common.Entities.Job job)
{
try
{
_notificationServiceBusClient.SendFinishedMessage(job, jobStatus, jobProcessResult.Output);
}
catch (Exception ex)
{
Geres.Diagnostics.GeresEventSource.Log.EngineJobHostJobProcessorFailedSendingServiceBusMessage(
RoleEnvironment.CurrentRoleInstance.Id,
RoleEnvironment.DeploymentId,
GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_JOBSTATUS,
GlobalConstants.SERVICEBUS_INTERNAL_SUBSCRIPTION_JOBFINISHED,
ex.Message,
ex.ToString()
);
}
}
public bool Process(string dedicatedBatchId)
{
CloudQueue queue = null;
OperationContext getMessageContext = new OperationContext();
OperationContext deleteMessageContext = new OperationContext();
var jobStatus = JobStatus.Submitted;
var jobOutput = string.Empty;
JobProcessResult jobProcessResult = new JobProcessResult();
//
// Get the list of batches (equivalet to queues) from table storage and order them based on their priority
//
var batches = GetPrioritizedListOfBatches(dedicatedBatchId);
//
// Find a message in the list of batches and process the first message found
//
BatchEntity singleBatch = null;
CloudQueueMessage message = null;
foreach (var batch in batches)
{
// Create the queue client.
queue = _queueClient.GetQueueReference(batch.Id);
//
// Try getting a message from the queue
//
try
{
message = TryDequeueMessage(getMessageContext, batch.Id, queue);
if (message != null)
{
singleBatch = batch;
GeresEventSource.Log.EngineJobHostReceivedMessage(message.AsString);
break;
}
}
catch (Exception ex)
{
message = null;
GeresEventSource.Log.EngineJobHostFailedDequeueMessage(batch.Id, batch.Name, ex.Message, ex.StackTrace);
break;
}
}
// Stop processing if there is no message
if (message == null)
{
GeresEventSource.Log.EngineJobHostNoMessageAvailable();
return(false);
}
//
// If there is a message available for processing, try lookup the job for it
//
Geres.Common.Entities.Job job = GetJob(message.AsString, singleBatch.Id);
if (job == null)
{
GeresEventSource.Log.EngineJobHostJobForMessageNotFound(message.AsString, singleBatch.Id);
// Nevertheless return true since the Processor dequeued a message
return(true);
}
else if ((job.Status == JobStatus.InProgress) || (job.Status == JobStatus.Started))
{
// The job is running, already, on another worker ... skip the job from being executed again
Geres.Diagnostics.GeresEventSource.Log.EngineJobHostSkipRunningJobSinceItIsStartedAlready(job.JobId, job.JobType, singleBatch.Id);
// Return true since we dequeued a message from the queue
return(true);
}
else
{
// Job found, send started message since everything below counts to job processing, already
TrySendStartServiceBusMessage(job);
}
//
// Try deploying the tenant and the code to process the job to an area on the machine
// If the path is null then the tenant failed to deploy
//
var tenantJobContext = TryDeployTenant(job, singleBatch.Id);
if (tenantJobContext == null)
{
return(true);
}
//
// Looking up the built-in job processor. There should always be a built-in job processor
// since the last fall-back will be the job processor worker process
//
GeresEventSource.Log.EngineJobHostLookingUpProcessor(string.Format("built-in job processor selection for {0}", job.JobType));
IJobImplementation processor = _builtInJobsFactory.Lookup(job);
if (processor == null)
{
GeresEventSource.Log.EngineJobHostProcessorLookupFailed("no built-in job processor returned - BUG in system!!");
return(true);
}
else
{
// Initialize the built-in job if it supports initialization
if (processor is IJobBuiltInImplementation)
{
((IJobBuiltInImplementation)processor).InitializeContextBeforeExecution
(
new BuiltInJobInitializationContext()
{
ExecutionAsUserName = tenantJobContext.UserName,
ExecutionAsUserPassword = tenantJobContext.UserPassword
}
);
}
}
//
// Job Processor created successfully, hence continue processing
// Setup a service-bus subscription to allow client-side cancellation
//
SubscriptionClient cancellationSubscriptionClient = null;
string cancellationSubscriptionName = Guid.NewGuid().ToString();
if (_singleJobCancellationEnabled)
{
try
{
cancellationSubscriptionClient = _cancellationServiceBusClient.CreateSubscription(job.JobId, cancellationSubscriptionName);
cancellationSubscriptionClient.OnMessage((receivedMessage) =>
{
processor.CancelProcessCallback();
});
}
catch (Exception ex)
{
Geres.Diagnostics.GeresEventSource.Log.EngineJobHostFailedSettingUpCancellationSubscriptionForJob(job.JobId, job.JobType, singleBatch.Id, ex.Message, ex.ToString());
// Set the job to aborted in the job log
try
{
SetJobMonitoringStatus
(
job.JobId, singleBatch.Id, JobStatus.AbortedInternalError, string.Empty, true
);
}
catch (Exception exi)
{
Geres.Diagnostics.GeresEventSource.Log.EngineJobHostFailedUpdatingJobStatus(job.JobId, job.JobName, job.JobType, exi.Message, exi.ToString());
}
// Job has been dequeued, so return true
return(true);
}
}
//
// Now try processing the job
//
try
{
// Update the status for the job
SetJobMonitoringStatus(job.JobId, singleBatch.Id, JobStatus.Started, string.Empty, true);
// simple flag so that the job status table is not constantly updated, i.e. update it once.
var updateProgress = true;
GeresEventSource.Log.EngineJobHostStartingJobProcessor(job.JobId, job.JobType, singleBatch.Id);
// use the newly discovered processor implementation to do the actual work.
// the callback provides the 3rd party code with the ability to provide progress updates back to the client
jobProcessResult = processor.DoWork(job, tenantJobContext.JobRootPath, tenantJobContext.JobWorkingRootPath,
(unit) =>
{
TrySendProgressServiceBusMessage(job, unit);
if (updateProgress)
{
SetJobMonitoringStatus(job.JobId, singleBatch.Id, JobStatus.InProgress, string.Empty, true);
updateProgress = false;
}
});
// Job processing completed, update the status
jobStatus = jobProcessResult.Status;
jobOutput = jobProcessResult.Output;
GeresEventSource.Log.EngineJobHostJobProcessorImplementationCompletedWithStatus(job.JobId, job.JobType, jobStatus.ToString());
}
catch (Exception ex)
{
// Job Processing failed
jobStatus = JobStatus.Failed;
jobOutput = string.Format("Job did run into uncaught exception: {0}!", ex.Message);
GeresEventSource.Log.EngineJobHostJobProcessorImplementationFailed(job.JobId, job.JobType, singleBatch.Id, ex.Message, ex.StackTrace);
}
//
// notify the originator of the job that processing has finished
//
TrySendServiceBusFinishedMessage(jobStatus, jobProcessResult, job);
// update the status of the job
try
{
SetJobMonitoringStatus(job.JobId, singleBatch.Id, jobStatus, jobOutput, false);
}
catch (Exception ex)
{
GeresEventSource.Log.EngineJobHostFailedUpdatingJobStatus(job.JobId, job.JobName, job.JobType, ex.Message, ex.StackTrace);
}
// delete the cancellation subscription for this jobId
if (_singleJobCancellationEnabled)
{
TryDeleteCacellationServiceBusSubscription(cancellationSubscriptionName, job.JobId, job.JobType);
}
// remove the job directory
_tenantManager.DeleteJobDirectory(job, singleBatch.Id);
// If the job has any other status but failed, delete the message
// If the job status is failed and it got dequeued too often, also delete the message
if (jobStatus != JobStatus.Failed)
{
TryDeleteMessage(singleBatch.Id, queue, message);
}
else if (message.DequeueCount >= _messageMaxRetryAttempts)
{
TryDeleteMessage(singleBatch.Id, queue, message);
}
return(true);
}
