public bool Process(string dedicatedBatchId)
{
CloudQueue queue = null;
OperationContext getMessageContext = new OperationContext();
OperationContext deleteMessageContext = new OperationContext();
var jobStatus = JobStatus.Submitted;
var jobOutput = string.Empty;
JobProcessResult jobProcessResult = new JobProcessResult();
//
// Get the list of batches (equivalet to queues) from table storage and order them based on their priority
//
var batches = GetPrioritizedListOfBatches(dedicatedBatchId);
//
// Find a message in the list of batches and process the first message found
//
BatchEntity singleBatch = null;
CloudQueueMessage message = null;
foreach (var batch in batches)
{
// Create the queue client.
queue = _queueClient.GetQueueReference(batch.Id);
//
// Try getting a message from the queue
//
try
{
message = TryDequeueMessage(getMessageContext, batch.Id, queue);
if (message != null)
{
singleBatch = batch;
GeresEventSource.Log.EngineJobHostReceivedMessage(message.AsString);
break;
}
}
catch (Exception ex)
{
message = null;
GeresEventSource.Log.EngineJobHostFailedDequeueMessage(batch.Id, batch.Name, ex.Message, ex.StackTrace);
break;
}
}
// Stop processing if there is no message
if (message == null)
{
GeresEventSource.Log.EngineJobHostNoMessageAvailable();
return(false);
}
//
// If there is a message available for processing, try lookup the job for it
//
Geres.Common.Entities.Job job = GetJob(message.AsString, singleBatch.Id);
if (job == null)
{
GeresEventSource.Log.EngineJobHostJobForMessageNotFound(message.AsString, singleBatch.Id);
// Nevertheless return true since the Processor dequeued a message
return(true);
}
else if ((job.Status == JobStatus.InProgress) || (job.Status == JobStatus.Started))
{
// The job is running, already, on another worker ... skip the job from being executed again
Geres.Diagnostics.GeresEventSource.Log.EngineJobHostSkipRunningJobSinceItIsStartedAlready(job.JobId, job.JobType, singleBatch.Id);
// Return true since we dequeued a message from the queue
return(true);
}
else
{
// Job found, send started message since everything below counts to job processing, already
TrySendStartServiceBusMessage(job);
}
//
// Try deploying the tenant and the code to process the job to an area on the machine
// If the path is null then the tenant failed to deploy
//
var tenantJobContext = TryDeployTenant(job, singleBatch.Id);
if (tenantJobContext == null)
{
return(true);
}
//
// Looking up the built-in job processor. There should always be a built-in job processor
// since the last fall-back will be the job processor worker process
//
GeresEventSource.Log.EngineJobHostLookingUpProcessor(string.Format("built-in job processor selection for {0}", job.JobType));
IJobImplementation processor = _builtInJobsFactory.Lookup(job);
if (processor == null)
{
GeresEventSource.Log.EngineJobHostProcessorLookupFailed("no built-in job processor returned - BUG in system!!");
return(true);
}
else
{
// Initialize the built-in job if it supports initialization
if (processor is IJobBuiltInImplementation)
{
((IJobBuiltInImplementation)processor).InitializeContextBeforeExecution
(
new BuiltInJobInitializationContext()
{
ExecutionAsUserName = tenantJobContext.UserName,
ExecutionAsUserPassword = tenantJobContext.UserPassword
}
);
}
}
//
// Job Processor created successfully, hence continue processing
// Setup a service-bus subscription to allow client-side cancellation
//
SubscriptionClient cancellationSubscriptionClient = null;
string cancellationSubscriptionName = Guid.NewGuid().ToString();
if (_singleJobCancellationEnabled)
{
try
{
cancellationSubscriptionClient = _cancellationServiceBusClient.CreateSubscription(job.JobId, cancellationSubscriptionName);
cancellationSubscriptionClient.OnMessage((receivedMessage) =>
{
processor.CancelProcessCallback();
});
}
catch (Exception ex)
{
Geres.Diagnostics.GeresEventSource.Log.EngineJobHostFailedSettingUpCancellationSubscriptionForJob(job.JobId, job.JobType, singleBatch.Id, ex.Message, ex.ToString());
// Set the job to aborted in the job log
try
{
SetJobMonitoringStatus
(
job.JobId, singleBatch.Id, JobStatus.AbortedInternalError, string.Empty, true
);
}
catch (Exception exi)
{
Geres.Diagnostics.GeresEventSource.Log.EngineJobHostFailedUpdatingJobStatus(job.JobId, job.JobName, job.JobType, exi.Message, exi.ToString());
}
// Job has been dequeued, so return true
return(true);
}
}
//
// Now try processing the job
//
try
{
// Update the status for the job
SetJobMonitoringStatus(job.JobId, singleBatch.Id, JobStatus.Started, string.Empty, true);
// simple flag so that the job status table is not constantly updated, i.e. update it once.
var updateProgress = true;
GeresEventSource.Log.EngineJobHostStartingJobProcessor(job.JobId, job.JobType, singleBatch.Id);
// use the newly discovered processor implementation to do the actual work.
// the callback provides the 3rd party code with the ability to provide progress updates back to the client
jobProcessResult = processor.DoWork(job, tenantJobContext.JobRootPath, tenantJobContext.JobWorkingRootPath,
(unit) =>
{
TrySendProgressServiceBusMessage(job, unit);
if (updateProgress)
{
SetJobMonitoringStatus(job.JobId, singleBatch.Id, JobStatus.InProgress, string.Empty, true);
updateProgress = false;
}
});
// Job processing completed, update the status
jobStatus = jobProcessResult.Status;
jobOutput = jobProcessResult.Output;
GeresEventSource.Log.EngineJobHostJobProcessorImplementationCompletedWithStatus(job.JobId, job.JobType, jobStatus.ToString());
}
catch (Exception ex)
{
// Job Processing failed
jobStatus = JobStatus.Failed;
jobOutput = string.Format("Job did run into uncaught exception: {0}!", ex.Message);
GeresEventSource.Log.EngineJobHostJobProcessorImplementationFailed(job.JobId, job.JobType, singleBatch.Id, ex.Message, ex.StackTrace);
}
//
// notify the originator of the job that processing has finished
//
TrySendServiceBusFinishedMessage(jobStatus, jobProcessResult, job);
// update the status of the job
try
{
SetJobMonitoringStatus(job.JobId, singleBatch.Id, jobStatus, jobOutput, false);
}
catch (Exception ex)
{
GeresEventSource.Log.EngineJobHostFailedUpdatingJobStatus(job.JobId, job.JobName, job.JobType, ex.Message, ex.StackTrace);
}
// delete the cancellation subscription for this jobId
if (_singleJobCancellationEnabled)
{
TryDeleteCacellationServiceBusSubscription(cancellationSubscriptionName, job.JobId, job.JobType);
}
// remove the job directory
_tenantManager.DeleteJobDirectory(job, singleBatch.Id);
// If the job has any other status but failed, delete the message
// If the job status is failed and it got dequeued too often, also delete the message
if (jobStatus != JobStatus.Failed)
{
TryDeleteMessage(singleBatch.Id, queue, message);
}
else if (message.DequeueCount >= _messageMaxRetryAttempts)
{
TryDeleteMessage(singleBatch.Id, queue, message);
}
return(true);
}
static int Main(string[] args)
{
try
{
//
// Validate parameters
//
var workingDirectory = Environment.GetEnvironmentVariable(JobWorkerProcessConstants.JOB_ENVIRONMENT_WORK_PATH);
var executingDirectory = Environment.GetEnvironmentVariable(JobWorkerProcessConstants.JOB_ENVIRONMENT_EXEC_PATH);
if (string.IsNullOrEmpty(workingDirectory))
{
LogError("Missing environment variable for working directory!");
return((int)JobStatus.AbortedInternalError);
}
if (string.IsNullOrEmpty(executingDirectory))
{
LogError("Missing environment variable for executing directory!");
return((int)JobStatus.AbortedInternalError);
}
//
// Get the current working directory in which this executable has been started
//
Log("Worker process started");
Log("Worker process executing directory {0}!", executingDirectory);
Log("Worker process working directory {0}!", workingDirectory);
//
// Deserialize the Job-description from the filesystem which should be in the path of execution
//
var jobPath = Path.Combine(executingDirectory, JobWorkerProcessConstants.JOB_XML_FILE);
Job jobToProcess = null;
try
{
using (var fs = new FileStream(jobPath, FileMode.Open, FileAccess.Read))
{
var serializer = new XmlSerializer(typeof(Job));
jobToProcess = (Job)serializer.Deserialize(fs);
}
}
catch (Exception ex)
{
LogError(string.Format("Unable to load jobs file {0}", jobPath), ex);
return((int)JobStatus.AbortedInternalError);
}
//
// Now resolve the job processor using the composition factory
//
Log("Loading job processor for job {0} with jobType {1}!", jobToProcess.JobId, jobToProcess.JobType);
IJobImplementation jobImplementation = null;
try
{
var jobFactory = new CompositionJobFactory(executingDirectory);
jobImplementation = jobFactory.Lookup(jobToProcess);
}
catch (TypeLoadException ex)
{
LogError(string.Format("Unable to load job implementation for jobId={0} with jobType={1} - TypeLoadExcpetion for type {2}!", jobToProcess.JobId, jobToProcess.JobType, ex.TypeName), ex);
return((int)JobStatus.AbortedJobProcessorMissingOrFailedLoading);
}
catch (System.Reflection.ReflectionTypeLoadException ex)
{
LogError(string.Format("Unable to load job implementation for jobId={0} with jobType={1} with ReflectionTypeLoadException!", jobToProcess.JobId, jobToProcess.JobType), ex);
foreach (var tex in ex.LoaderExceptions)
{
LogError("- Loader Exception:", tex);
}
return((int)JobStatus.AbortedJobProcessorMissingOrFailedLoading);
}
catch (Exception ex)
{
LogError(string.Format("Unable to load job implementation for jobId={0} with jobType={1}!", jobToProcess.JobId, jobToProcess.JobType), ex);
return((int)JobStatus.AbortedJobProcessorMissingOrFailedLoading);
}
// Now execute the job itself
Log("Start processing job...");
try
{
var jobResult = jobImplementation.DoWork
(
jobToProcess,
executingDirectory,
workingDirectory,
progress =>
{
LogProgress(progress);
}
);
// Log that the work-implementation completed without an exception
Log("Job implementation completed successfully with status {0}", jobResult.Status.ToString());
// Log the output so that it's recorded by the system and return the status from the job execution itself
Log(jobResult.Output);
return((int)jobResult.Status);
}
catch (Exception ex)
{
LogError("Job ran into an unhandled exception!", ex);
return((int)JobStatus.FailedUnexpectedly);
}
}
catch (Exception ex)
{
LogError("An unhandled error occured in the job worker process!", ex);
return((int)JobStatus.AbortedInternalError);
}
}
