Ejemplo n.º 1
0
 private void TrySendProgressServiceBusMessage(Geres.Common.Entities.Job job, string progress)
 {
     try
     {
         _notificationServiceBusClient.SendProgressMessage(job, progress);
     }
     catch (Exception ex)
     {
         Geres.Diagnostics.GeresEventSource.Log.EngineJobHostJobProcessorFailedSendingServiceBusMessage(
             RoleEnvironment.CurrentRoleInstance.Id,
             RoleEnvironment.DeploymentId,
             GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_JOBSTATUS,
             GlobalConstants.SERVICEBUS_INTERNAL_SUBSCRIPTION_JOBPROGRESS,
             ex.Message,
             ex.ToString()
             );
     }
 }
Ejemplo n.º 2
0
 private void TrySendServiceBusFinishedMessage(JobStatus jobStatus, JobProcessResult jobProcessResult, Geres.Common.Entities.Job job)
 {
     try
     {
         _notificationServiceBusClient.SendFinishedMessage(job, jobStatus, jobProcessResult.Output);
     }
     catch (Exception ex)
     {
         Geres.Diagnostics.GeresEventSource.Log.EngineJobHostJobProcessorFailedSendingServiceBusMessage(
             RoleEnvironment.CurrentRoleInstance.Id,
             RoleEnvironment.DeploymentId,
             GlobalConstants.SERVICEBUS_INTERNAL_TOPICS_JOBSTATUS,
             GlobalConstants.SERVICEBUS_INTERNAL_SUBSCRIPTION_JOBFINISHED,
             ex.Message,
             ex.ToString()
             );
     }
 }
Ejemplo n.º 3
0
        public bool Process(string dedicatedBatchId)
        {
            CloudQueue       queue                = null;
            OperationContext getMessageContext    = new OperationContext();
            OperationContext deleteMessageContext = new OperationContext();

            var jobStatus = JobStatus.Submitted;
            var jobOutput = string.Empty;
            JobProcessResult jobProcessResult = new JobProcessResult();

            //
            // Get the list of batches (equivalet to queues) from table storage and order them based on their priority
            //
            var batches = GetPrioritizedListOfBatches(dedicatedBatchId);

            //
            // Find a message in the list of batches and process the first message found
            //
            BatchEntity       singleBatch = null;
            CloudQueueMessage message     = null;

            foreach (var batch in batches)
            {
                // Create the queue client.
                queue = _queueClient.GetQueueReference(batch.Id);

                //
                // Try getting a message from the queue
                //
                try
                {
                    message = TryDequeueMessage(getMessageContext, batch.Id, queue);
                    if (message != null)
                    {
                        singleBatch = batch;
                        GeresEventSource.Log.EngineJobHostReceivedMessage(message.AsString);
                        break;
                    }
                }
                catch (Exception ex)
                {
                    message = null;
                    GeresEventSource.Log.EngineJobHostFailedDequeueMessage(batch.Id, batch.Name, ex.Message, ex.StackTrace);
                    break;
                }
            }
            // Stop processing if there is no message
            if (message == null)
            {
                GeresEventSource.Log.EngineJobHostNoMessageAvailable();
                return(false);
            }

            //
            // If there is a message available for processing, try lookup the job for it
            //
            Geres.Common.Entities.Job job = GetJob(message.AsString, singleBatch.Id);
            if (job == null)
            {
                GeresEventSource.Log.EngineJobHostJobForMessageNotFound(message.AsString, singleBatch.Id);
                // Nevertheless return true since the Processor dequeued a message
                return(true);
            }
            else if ((job.Status == JobStatus.InProgress) || (job.Status == JobStatus.Started))
            {
                // The job is running, already, on another worker ... skip the job from being executed again
                Geres.Diagnostics.GeresEventSource.Log.EngineJobHostSkipRunningJobSinceItIsStartedAlready(job.JobId, job.JobType, singleBatch.Id);
                // Return true since we dequeued a message from the queue
                return(true);
            }
            else
            {
                // Job found, send started message since everything below counts to job processing, already
                TrySendStartServiceBusMessage(job);
            }

            //
            // Try deploying the tenant and the code to process the job to an area on the machine
            // If the path is null then the tenant failed to deploy
            //
            var tenantJobContext = TryDeployTenant(job, singleBatch.Id);

            if (tenantJobContext == null)
            {
                return(true);
            }

            //
            // Looking up the built-in job processor. There should always be a built-in job processor
            // since the last fall-back will be the job processor worker process
            //
            GeresEventSource.Log.EngineJobHostLookingUpProcessor(string.Format("built-in job processor selection for {0}", job.JobType));
            IJobImplementation processor = _builtInJobsFactory.Lookup(job);

            if (processor == null)
            {
                GeresEventSource.Log.EngineJobHostProcessorLookupFailed("no built-in job processor returned - BUG in system!!");
                return(true);
            }
            else
            {
                // Initialize the built-in job if it supports initialization
                if (processor is IJobBuiltInImplementation)
                {
                    ((IJobBuiltInImplementation)processor).InitializeContextBeforeExecution
                    (
                        new BuiltInJobInitializationContext()
                    {
                        ExecutionAsUserName     = tenantJobContext.UserName,
                        ExecutionAsUserPassword = tenantJobContext.UserPassword
                    }
                    );
                }
            }

            //
            // Job Processor created successfully, hence continue processing
            // Setup a service-bus subscription to allow client-side cancellation
            //
            SubscriptionClient cancellationSubscriptionClient = null;
            string             cancellationSubscriptionName   = Guid.NewGuid().ToString();

            if (_singleJobCancellationEnabled)
            {
                try
                {
                    cancellationSubscriptionClient = _cancellationServiceBusClient.CreateSubscription(job.JobId, cancellationSubscriptionName);
                    cancellationSubscriptionClient.OnMessage((receivedMessage) =>
                    {
                        processor.CancelProcessCallback();
                    });
                }
                catch (Exception ex)
                {
                    Geres.Diagnostics.GeresEventSource.Log.EngineJobHostFailedSettingUpCancellationSubscriptionForJob(job.JobId, job.JobType, singleBatch.Id, ex.Message, ex.ToString());

                    // Set the job to aborted in the job log
                    try
                    {
                        SetJobMonitoringStatus
                        (
                            job.JobId, singleBatch.Id, JobStatus.AbortedInternalError, string.Empty, true
                        );
                    }
                    catch (Exception exi)
                    {
                        Geres.Diagnostics.GeresEventSource.Log.EngineJobHostFailedUpdatingJobStatus(job.JobId, job.JobName, job.JobType, exi.Message, exi.ToString());
                    }

                    // Job has been dequeued, so return true
                    return(true);
                }
            }

            //
            // Now try processing the job
            //
            try
            {
                // Update the status for the job
                SetJobMonitoringStatus(job.JobId, singleBatch.Id, JobStatus.Started, string.Empty, true);

                // simple flag so that the job status table is not constantly updated, i.e. update it once.
                var updateProgress = true;

                GeresEventSource.Log.EngineJobHostStartingJobProcessor(job.JobId, job.JobType, singleBatch.Id);
                // use the newly discovered processor implementation to do the actual work.
                // the callback provides the 3rd party code with the ability to provide progress updates back to the client
                jobProcessResult = processor.DoWork(job, tenantJobContext.JobRootPath, tenantJobContext.JobWorkingRootPath,
                                                    (unit) =>
                {
                    TrySendProgressServiceBusMessage(job, unit);

                    if (updateProgress)
                    {
                        SetJobMonitoringStatus(job.JobId, singleBatch.Id, JobStatus.InProgress, string.Empty, true);
                        updateProgress = false;
                    }
                });

                // Job processing completed, update the status
                jobStatus = jobProcessResult.Status;
                jobOutput = jobProcessResult.Output;

                GeresEventSource.Log.EngineJobHostJobProcessorImplementationCompletedWithStatus(job.JobId, job.JobType, jobStatus.ToString());
            }
            catch (Exception ex)
            {
                // Job Processing failed
                jobStatus = JobStatus.Failed;
                jobOutput = string.Format("Job did run into uncaught exception: {0}!", ex.Message);
                GeresEventSource.Log.EngineJobHostJobProcessorImplementationFailed(job.JobId, job.JobType, singleBatch.Id, ex.Message, ex.StackTrace);
            }

            //
            // notify the originator of the job that processing has finished
            //
            TrySendServiceBusFinishedMessage(jobStatus, jobProcessResult, job);

            // update the status of the job
            try
            {
                SetJobMonitoringStatus(job.JobId, singleBatch.Id, jobStatus, jobOutput, false);
            }
            catch (Exception ex)
            {
                GeresEventSource.Log.EngineJobHostFailedUpdatingJobStatus(job.JobId, job.JobName, job.JobType, ex.Message, ex.StackTrace);
            }

            // delete the cancellation subscription for this jobId
            if (_singleJobCancellationEnabled)
            {
                TryDeleteCacellationServiceBusSubscription(cancellationSubscriptionName, job.JobId, job.JobType);
            }

            // remove the job directory
            _tenantManager.DeleteJobDirectory(job, singleBatch.Id);

            // If the job has any other status but failed, delete the message
            // If the job status is failed and it got dequeued too often, also delete the message
            if (jobStatus != JobStatus.Failed)
            {
                TryDeleteMessage(singleBatch.Id, queue, message);
            }
            else if (message.DequeueCount >= _messageMaxRetryAttempts)
            {
                TryDeleteMessage(singleBatch.Id, queue, message);
            }

            return(true);
        }