Exemple #1
0
 internal static void LogResult(JobResult result, ILogger logger, string jobName) {
     if (result != null) {
         if (result.IsCancelled)
             logger.Warn(result.Error, "Job run \"{0}\" cancelled: {1}", jobName, result.Message);
         else if (!result.IsSuccess)
             logger.Error(result.Error, "Job run \"{0}\" failed: {1}", jobName, result.Message);
         else if (!String.IsNullOrEmpty(result.Message))
             logger.Info("Job run \"{0}\" succeeded: {1}", jobName, result.Message);
         else
             logger.Trace("Job run \"{0}\" succeeded.", jobName);
     } else {
         logger.Error("Null job run result for \"{0}\".", jobName);
     }
 }
Exemple #2
0
        public async Task <JobResult> RunAsync(CancellationToken cancellationToken = default(CancellationToken))
        {
            IQueueEntry <WorkItemData> queueEntry;

            using (var timeoutCancellationTokenSource = new CancellationTokenSource(30000))
                using (var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellationTokenSource.Token)) {
                    try {
                        queueEntry = await _queue.DequeueAsync(linkedCancellationTokenSource.Token).AnyContext();
                    } catch (Exception ex) {
                        return(JobResult.FromException(ex, $"Error trying to dequeue work item: {ex.Message}"));
                    }
                }

            return(await ProcessAsync(queueEntry, cancellationToken).AnyContext());
        }
Exemple #3
0
        public async Task <JobResult> RunAsync(Type jobType, TimeSpan?initialDelay = null, CancellationToken cancellationToken = default(CancellationToken))
        {
            if (initialDelay.HasValue && initialDelay.Value > TimeSpan.Zero)
            {
                await Task.Delay(initialDelay.Value, cancellationToken).AnyContext();
            }

            var jobInstance = CreateJobInstance(jobType);

            if (jobInstance == null)
            {
                return(JobResult.FailedWithMessage("Could not create job instance: " + jobType));
            }

            return(await jobInstance.RunAsync(cancellationToken).AnyContext());
        }
Exemple #4
0
        public Task <JobResult> RunAsync(CancellationToken cancellationToken = default(CancellationToken))
        {
            try {
                var lockValue = GetJobLock();
                if (lockValue == null)
                {
                    Log.Warn().Message("Unable to acquire job lock").Write();
                    return(Task.FromResult(JobResult.FailedWithMessage("Unable to acquire job lock.")));
                }

                using (lockValue)
                    return(TryRunAsync(cancellationToken));
            } catch (TimeoutException) {
                return(Task.FromResult(JobResult.FailedWithMessage("Timeout attempting to acquire lock.")));
            }
        }
Exemple #5
0
        public async Task <JobResult> RunAsync(CancellationToken cancellationToken = default(CancellationToken))
        {
            IDisposable scope = new EmptyDisposable();

            if (!_runningContinuous)
            {
                scope = _logger.BeginScope(s => s.Property("job", _jobName));
            }

            using (scope) {
                _logger.Trace("Job run \"{0}\" starting...", _jobName);

                using (var lockValue = await GetJobLockAsync().AnyContext()) {
                    if (lockValue == null)
                    {
                        return(JobResult.SuccessWithMessage("Unable to acquire job lock."));
                    }

                    var result = await TryRunAsync(new JobRunContext(lockValue, cancellationToken)).AnyContext();

                    if (result != null)
                    {
                        if (!result.IsSuccess)
                        {
                            _logger.Error(result.Error, "Job run \"{0}\" failed: {1}", GetType().Name, result.Message);
                        }
                        else if (!String.IsNullOrEmpty(result.Message))
                        {
                            _logger.Info("Job run \"{0}\" succeeded: {1}", GetType().Name, result.Message);
                        }
                        else
                        {
                            _logger.Trace("Job run \"{0}\" succeeded.", GetType().Name);
                        }
                    }
                    else
                    {
                        _logger.Error("Null job run result for \"{0}\".", GetType().Name);
                    }

                    return(result);
                }
            }
        }
Exemple #6
0
 internal static void LogResult(JobResult result, ILogger logger, string jobName)
 {
     if (result != null)
     {
         if (!result.IsSuccess)
         {
             logger.Error(result.Error, "Job run \"{0}\" failed: {1}", jobName, result.Message);
         }
         else if (!String.IsNullOrEmpty(result.Message))
         {
             logger.Info("Job run \"{0}\" succeeded: {1}", jobName, result.Message);
         }
         else
         {
             logger.Trace("Job run \"{0}\" succeeded.", jobName);
         }
     }
     else
     {
         logger.Error("Null job run result for \"{0}\".", jobName);
     }
 }
Exemple #7
0
        public async Task <JobResult> RunAsync(CancellationToken cancellationToken = default(CancellationToken))
        {
            EnsureJobNameSet();
            Logger.Trace().Logger(_jobName).Message("Job run \"{0}\" starting...", _jobName).Write();

            using (var lockValue = await GetJobLockAsync().AnyContext()) {
                if (lockValue == null)
                {
                    return(JobResult.SuccessWithMessage("Unable to acquire job lock."));
                }

                var result = await TryRunAsync(new JobRunContext(lockValue, cancellationToken)).AnyContext();

                if (result != null)
                {
                    if (!result.IsSuccess)
                    {
                        Logger.Error().Logger(_jobName).Message("Job run \"{0}\" failed: {1}", GetType().Name, result.Message).Exception(result.Error).Write();
                    }
                    else if (!String.IsNullOrEmpty(result.Message))
                    {
                        Logger.Info().Logger(_jobName).Message("Job run \"{0}\" succeeded: {1}", GetType().Name, result.Message).Write();
                    }
                    else
                    {
                        Logger.Trace().Logger(_jobName).Message("Job run \"{0}\" succeeded.", GetType().Name).Write();
                    }
                }
                else
                {
                    Logger.Error().Logger(_jobName).Message("Null job run result for \"{0}\".", GetType().Name).Write();
                }

                return(result);
            }
        }
Exemple #8
0
        public async Task <JobResult> RunAsync(CancellationToken cancellationToken = default(CancellationToken))
        {
            var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, TimeSpan.FromSeconds(30).ToCancellationToken());

            IQueueEntry <WorkItemData> queueEntry;

            try {
                queueEntry = await _queue.DequeueAsync(linkedCancellationTokenSource.Token).AnyContext();
            } catch (Exception ex) {
                return(JobResult.FromException(ex, $"Error trying to dequeue work item: {ex.Message}"));
            }

            if (queueEntry == null)
            {
                return(JobResult.Success);
            }

            if (cancellationToken.IsCancellationRequested)
            {
                _logger.Info("Job was cancelled. Abandoning {queueEntryType} work item: {queueEntryId}", queueEntry.Value.Type, queueEntry.Id);
                await queueEntry.AbandonAsync().AnyContext();

                return(JobResult.CancelledWithMessage($"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}"));
            }

            Type workItemDataType = null;

            try {
                workItemDataType = Type.GetType(queueEntry.Value.Type);
            } catch (Exception ex) {
                await queueEntry.AbandonAsync().AnyContext();

                return(JobResult.FromException(ex, $"Could not resolve {workItemDataType.Name} work item data type."));
            }

            if (workItemDataType == null)
            {
                await queueEntry.AbandonAsync().AnyContext();

                return(JobResult.FailedWithMessage($"Could not resolve {workItemDataType.Name} work item data type."));
            }

            object workItemData;

            try {
                workItemData = await _queue.Serializer.DeserializeAsync(queueEntry.Value.Data, workItemDataType).AnyContext();
            } catch (Exception ex) {
                await queueEntry.AbandonAsync().AnyContext();

                return(JobResult.FromException(ex, $"Failed to parse {workItemDataType.Name} work item data."));
            }

            var handler = _handlers.GetHandler(workItemDataType);

            if (handler == null)
            {
                await queueEntry.CompleteAsync().AnyContext();

                return(JobResult.FailedWithMessage($"Handler for type {workItemDataType.Name} not registered."));
            }

            if (queueEntry.Value.SendProgressReports)
            {
                await _messageBus.PublishAsync(new WorkItemStatus {
                    WorkItemId = queueEntry.Value.WorkItemId,
                    Progress   = 0,
                    Type       = queueEntry.Value.Type
                }).AnyContext();
            }

            var lockValue = await handler.GetWorkItemLockAsync(workItemData, cancellationToken).AnyContext();

            if (lockValue == null)
            {
                await queueEntry.AbandonAsync().AnyContext();

                handler.Log.Trace("Unable to acquire work item lock.");
                return(JobResult.Success);
            }

            var progressCallback = new Func <int, string, Task>(async(progress, message) => {
                if (handler.AutoRenewLockOnProgress)
                {
                    await queueEntry.RenewLockAsync().AnyContext();
                }

                if (handler.AutoRenewLockOnProgress)
                {
                    await lockValue.RenewAsync().AnyContext();
                }

                await _messageBus.PublishAsync(new WorkItemStatus {
                    WorkItemId = queueEntry.Value.WorkItemId,
                    Progress   = progress,
                    Message    = message,
                    Type       = queueEntry.Value.Type
                }).AnyContext();
            });

            try {
                handler.LogProcessingQueueEntry(queueEntry, workItemDataType, workItemData);
                await handler.HandleItemAsync(new WorkItemContext(workItemData, JobId, lockValue, cancellationToken, progressCallback)).AnyContext();

                if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted)
                {
                    await queueEntry.CompleteAsync().AnyContext();

                    handler.LogAutoCompletedQueueEntry(queueEntry, workItemDataType, workItemData);
                }

                if (queueEntry.Value.SendProgressReports)
                {
                    await _messageBus.PublishAsync(new WorkItemStatus {
                        WorkItemId = queueEntry.Value.WorkItemId,
                        Progress   = 100,
                        Type       = queueEntry.Value.Type
                    }).AnyContext();
                }

                return(JobResult.Success);
            } catch (Exception ex) {
                if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted)
                {
                    await queueEntry.AbandonAsync().AnyContext();
                }

                handler.Log.Error(ex, "Error processing {0} work item queue entry ({1}).", workItemDataType.Name, queueEntry.Id);
                return(JobResult.FromException(ex, $"Error in handler {workItemDataType.Name}."));
            } finally {
                await lockValue.ReleaseAsync().AnyContext();
            }
        }
Exemple #9
0
        public async Task <JobResult> ProcessAsync(IQueueEntry <WorkItemData> queueEntry, CancellationToken cancellationToken)
        {
            if (queueEntry == null)
            {
                return(JobResult.Success);
            }

            if (cancellationToken.IsCancellationRequested)
            {
                await queueEntry.AbandonAsync().AnyContext();

                return(JobResult.CancelledWithMessage($"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}"));
            }

            Type workItemDataType;

            try {
                workItemDataType = Type.GetType(queueEntry.Value.Type);
            } catch (Exception ex) {
                await queueEntry.AbandonAsync().AnyContext();

                return(JobResult.FromException(ex, $"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}: Could not resolve work item data type."));
            }

            if (workItemDataType == null)
            {
                await queueEntry.AbandonAsync().AnyContext();

                return(JobResult.FailedWithMessage($"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}: Could not resolve work item data type."));
            }

            object workItemData;

            try {
                workItemData = _queue.Serializer.Deserialize(queueEntry.Value.Data, workItemDataType);
            } catch (Exception ex) {
                await queueEntry.AbandonAsync().AnyContext();

                return(JobResult.FromException(ex, $"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}: Failed to parse {workItemDataType.Name} work item data."));
            }

            var handler = _handlers.GetHandler(workItemDataType);

            if (handler == null)
            {
                await queueEntry.CompleteAsync().AnyContext();

                return(JobResult.FailedWithMessage($"Completing {queueEntry.Value.Type} work item: {queueEntry.Id}: Handler for type {workItemDataType.Name} not registered."));
            }

            if (queueEntry.Value.SendProgressReports)
            {
                await ReportProgressAsync(handler, queueEntry).AnyContext();
            }

            var lockValue = await handler.GetWorkItemLockAsync(workItemData, cancellationToken).AnyContext();

            if (lockValue == null)
            {
                if (handler.Log.IsEnabled(LogLevel.Information))
                {
                    handler.Log.LogInformation("Abandoning {TypeName} work item: {Id}: Unable to acquire work item lock.", queueEntry.Value.Type, queueEntry.Id);
                }

                await queueEntry.AbandonAsync().AnyContext();

                return(JobResult.Success);
            }

            var progressCallback = new Func <int, string, Task>(async(progress, message) => {
                if (handler.AutoRenewLockOnProgress)
                {
                    try {
                        await Task.WhenAll(
                            queueEntry.RenewLockAsync(),
                            lockValue.RenewAsync()
                            ).AnyContext();
                    } catch (Exception ex) {
                        if (handler.Log.IsEnabled(LogLevel.Error))
                        {
                            handler.Log.LogError(ex, "Error renewing work item locks: {Message}", ex.Message);
                        }
                    }
                }

                await ReportProgressAsync(handler, queueEntry, progress, message).AnyContext();
                if (handler.Log.IsEnabled(LogLevel.Information))
                {
                    handler.Log.LogInformation("{TypeName} Progress {Progress}%: {Message}", workItemDataType.Name, progress, message);
                }
            });

            try {
                handler.LogProcessingQueueEntry(queueEntry, workItemDataType, workItemData);
                await handler.HandleItemAsync(new WorkItemContext(workItemData, JobId, lockValue, cancellationToken, progressCallback)).AnyContext();

                if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted)
                {
                    await queueEntry.CompleteAsync().AnyContext();

                    handler.LogAutoCompletedQueueEntry(queueEntry, workItemDataType, workItemData);
                }

                if (queueEntry.Value.SendProgressReports)
                {
                    await ReportProgressAsync(handler, queueEntry, 100).AnyContext();
                }

                return(JobResult.Success);
            } catch (Exception ex) {
                if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted)
                {
                    await queueEntry.AbandonAsync().AnyContext();

                    return(JobResult.FromException(ex, $"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}: Error in handler {workItemDataType.Name}."));
                }

                return(JobResult.FromException(ex, $"Error processing {queueEntry.Value.Type} work item: {queueEntry.Id} in handler: {workItemDataType.Name}"));
            } finally {
                await lockValue.ReleaseAsync().AnyContext();
            }
        }
        public async Task <JobResult> ProcessAsync(IQueueEntry <T> queueEntry, CancellationToken cancellationToken)
        {
            if (queueEntry == null)
            {
                return(JobResult.Success);
            }

            using (_logger.BeginScope(s => s
                                      .Property("QueueEntryId", queueEntry.Id)
                                      .PropertyIf("CorrelationId", queueEntry.CorrelationId, !String.IsNullOrEmpty(queueEntry.CorrelationId))
                                      .Property("QueueEntryName", _queueEntryName))) {
                _logger.LogInformation("Processing queue entry: id={QueueEntryId} type={QueueEntryName} attempt={QueueEntryAttempt}", queueEntry.Id, _queueEntryName, queueEntry.Attempts);

                if (cancellationToken.IsCancellationRequested)
                {
                    if (_logger.IsEnabled(LogLevel.Information))
                    {
                        _logger.LogInformation("Job was cancelled. Abandoning {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id);
                    }

                    await queueEntry.AbandonAsync().AnyContext();

                    return(JobResult.CancelledWithMessage($"Abandoning {_queueEntryName} queue entry: {queueEntry.Id}"));
                }

                var lockValue = await GetQueueEntryLockAsync(queueEntry, cancellationToken).AnyContext();

                if (lockValue == null)
                {
                    await queueEntry.AbandonAsync().AnyContext();

                    _logger.LogTrace("Unable to acquire queue entry lock.");
                    return(JobResult.Success);
                }

                bool isTraceLogLevelEnabled = _logger.IsEnabled(LogLevel.Trace);
                try {
                    LogProcessingQueueEntry(queueEntry);
                    var result = await ProcessQueueEntryAsync(new QueueEntryContext <T>(queueEntry, lockValue, cancellationToken)).AnyContext();

                    if (!AutoComplete || queueEntry.IsCompleted || queueEntry.IsAbandoned)
                    {
                        return(result);
                    }

                    if (result.IsSuccess)
                    {
                        await queueEntry.CompleteAsync().AnyContext();

                        LogAutoCompletedQueueEntry(queueEntry);
                    }
                    else
                    {
                        if (result.Error != null || result.Message != null)
                        {
                            _logger.LogError(result.Error, "{QueueEntryName} queue entry {Id} returned an unsuccessful response: {ErrorMessage}", _queueEntryName, queueEntry.Id, result.Message ?? result.Error?.Message);
                        }

                        if (isTraceLogLevelEnabled)
                        {
                            _logger.LogTrace("Processing was not successful. Auto Abandoning {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id);
                        }
                        await queueEntry.AbandonAsync().AnyContext();

                        if (_logger.IsEnabled(LogLevel.Warning))
                        {
                            _logger.LogWarning("Auto abandoned {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id);
                        }
                    }

                    return(result);
                } catch (Exception ex) {
                    _logger.LogError(ex, "Error processing {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id);

                    if (!queueEntry.IsCompleted && !queueEntry.IsAbandoned)
                    {
                        await queueEntry.AbandonAsync().AnyContext();
                    }

                    throw;
                } finally {
                    if (isTraceLogLevelEnabled)
                    {
                        _logger.LogTrace("Releasing Lock for {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id);
                    }
                    await lockValue.ReleaseAsync().AnyContext();

                    if (isTraceLogLevelEnabled)
                    {
                        _logger.LogTrace("Released Lock for {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id);
                    }
                }
            }
        }
Exemple #11
0
        public async Task <JobResult> ProcessAsync(IQueueEntry <T> queueEntry, CancellationToken cancellationToken)
        {
            if (queueEntry == null)
            {
                return(JobResult.Success);
            }

            if (cancellationToken.IsCancellationRequested)
            {
                if (_logger.IsEnabled(LogLevel.Information))
                {
                    _logger.LogInformation("Job was cancelled. Abandoning {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id);
                }

                await queueEntry.AbandonAsync().AnyContext();

                return(JobResult.CancelledWithMessage($"Abandoning {_queueEntryName} queue entry: {queueEntry.Id}"));
            }

            var lockValue = await GetQueueEntryLockAsync(queueEntry, cancellationToken).AnyContext();

            if (lockValue == null)
            {
                await queueEntry.AbandonAsync().AnyContext();

                _logger.LogTrace("Unable to acquire queue entry lock.");
                return(JobResult.Success);
            }

            try {
                LogProcessingQueueEntry(queueEntry);
                var result = await ProcessQueueEntryAsync(new QueueEntryContext <T>(queueEntry, lockValue, cancellationToken)).AnyContext();

                if (!AutoComplete || queueEntry.IsCompleted || queueEntry.IsAbandoned)
                {
                    return(result);
                }

                if (result.IsSuccess)
                {
                    await queueEntry.CompleteAsync().AnyContext();

                    LogAutoCompletedQueueEntry(queueEntry);
                }
                else
                {
                    await queueEntry.AbandonAsync().AnyContext();

                    if (_logger.IsEnabled(LogLevel.Warning))
                    {
                        _logger.LogWarning("Auto abandoned {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id);
                    }
                }

                return(result);
            } catch (Exception ex) {
                if (_logger.IsEnabled(LogLevel.Error))
                {
                    _logger.LogError(ex, "Error processing {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id);
                }

                if (!queueEntry.IsCompleted && !queueEntry.IsAbandoned)
                {
                    await queueEntry.AbandonAsync().AnyContext();
                }

                throw;
            } finally {
                await lockValue.ReleaseAsync().AnyContext();
            }
        }
Exemple #12
0
        public async Task <JobResult> RunAsync(CancellationToken cancellationToken = new CancellationToken())
        {
            var linkedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, TimeSpan.FromSeconds(30).ToCancellationToken());

            IQueueEntry <T> queueEntry;

            try {
                queueEntry = await _queue.DequeueAsync(linkedCancellationToken.Token).AnyContext();
            } catch (Exception ex) {
                return(JobResult.FromException(ex, $"Error trying to dequeue message: {ex.Message}"));
            }

            if (queueEntry == null)
            {
                return(JobResult.Success);
            }

            if (cancellationToken.IsCancellationRequested)
            {
                _logger.Info(() => $"Job was cancelled. Abandoning {_queueEntryName} queue item: {queueEntry.Id}");
                await queueEntry.AbandonAsync().AnyContext();

                return(JobResult.CancelledWithMessage($"Abandoning {_queueEntryName} queue item: {queueEntry.Id}"));
            }

            var lockValue = await GetQueueEntryLockAsync(queueEntry, cancellationToken).AnyContext();

            if (lockValue == null)
            {
                await queueEntry.AbandonAsync().AnyContext();

                _logger.Trace("Unable to acquire queue entry lock.");
                return(JobResult.Success);
            }

            try {
                LogProcessingQueueEntry(queueEntry);
                var result = await ProcessQueueEntryAsync(new QueueEntryContext <T>(queueEntry, lockValue, cancellationToken)).AnyContext();

                if (!AutoComplete || queueEntry.IsCompleted || queueEntry.IsAbandoned)
                {
                    return(result);
                }

                if (result.IsSuccess)
                {
                    await queueEntry.CompleteAsync().AnyContext();

                    LogAutoCompletedQueueEntry(queueEntry);
                }
                else
                {
                    await queueEntry.AbandonAsync().AnyContext();

                    _logger.Warn(() => $"Auto abandoned {_queueEntryName} queue entry ({queueEntry.Id}).");
                }

                return(result);
            } catch (Exception ex) {
                _logger.Error(ex, () => $"Error processing {_queueEntryName} queue entry ({queueEntry.Id}).");
                if (!queueEntry.IsCompleted && !queueEntry.IsAbandoned)
                {
                    await queueEntry.AbandonAsync().AnyContext();
                }

                throw;
            } finally {
                await lockValue.ReleaseAsync().AnyContext();
            }
        }
Exemple #13
0
        protected async override Task <JobResult> ProcessQueueEntryAsync(JobQueueEntryContext <WorkItemData> context)
        {
            var workItemDataType = Type.GetType(context.QueueEntry.Value.Type);

            if (workItemDataType == null)
            {
                return(JobResult.FailedWithMessage("Could not resolve work item data type."));
            }

            object workItemData;

            try {
                workItemData = await _queue.Serializer.DeserializeAsync(context.QueueEntry.Value.Data, workItemDataType).AnyContext();
            } catch (Exception ex) {
                return(JobResult.FromException(ex, "Failed to parse work item data."));
            }

            var handler = _handlers.GetHandler(workItemDataType);

            if (handler == null)
            {
                return(JobResult.FailedWithMessage($"Handler for type {workItemDataType.Name} not registered."));
            }

            if (context.QueueEntry.Value.SendProgressReports)
            {
                await _messageBus.PublishAsync(new WorkItemStatus {
                    WorkItemId = context.QueueEntry.Value.WorkItemId,
                    Progress   = 0,
                    Type       = context.QueueEntry.Value.Type
                }).AnyContext();
            }

            using (var lockValue = await handler.GetWorkItemLockAsync(workItemData, context.CancellationToken).AnyContext()) {
                if (lockValue == null)
                {
                    return(JobResult.SuccessWithMessage("Unable to acquire work item lock."));
                }

                var progressCallback = new Func <int, string, Task>(async(progress, message) => {
                    if (handler.AutoRenewLockOnProgress && lockValue != null)
                    {
                        await lockValue.RenewAsync().AnyContext();
                    }

                    await _messageBus.PublishAsync(new WorkItemStatus {
                        WorkItemId = context.QueueEntry.Value.WorkItemId,
                        Progress   = progress,
                        Message    = message,
                        Type       = context.QueueEntry.Value.Type
                    }).AnyContext();
                });

                try {
                    _logger.Info("Processing {0} work item queue entry ({1}).", workItemDataType.Name, context.QueueEntry.Id);
                    await handler.HandleItemAsync(new WorkItemContext(context, workItemData, JobId, lockValue, progressCallback)).AnyContext();
                } catch (Exception ex) {
                    await context.QueueEntry.AbandonAsync().AnyContext();

                    _logger.Error("Error processing {0} work item queue entry ({1}).", workItemDataType.Name, context.QueueEntry.Id);

                    return(JobResult.FromException(ex, $"Error in handler {workItemDataType.Name}."));
                }

                await context.QueueEntry.CompleteAsync().AnyContext();

                _logger.Info("Completed {0} work item queue entry ({1}).", workItemDataType.Name, context.QueueEntry.Id);

                if (context.QueueEntry.Value.SendProgressReports)
                {
                    await _messageBus.PublishAsync(new WorkItemStatus {
                        WorkItemId = context.QueueEntry.Value.WorkItemId,
                        Progress   = 100,
                        Type       = context.QueueEntry.Value.Type
                    }).AnyContext();
                }

                return(JobResult.Success);
            }
        }
        protected override async Task <JobResult> RunInternalAsync(JobRunContext context)
        {
            var linkedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken, TimeSpan.FromSeconds(30).ToCancellationToken());

            IQueueEntry <T> queueEntry;

            try {
                queueEntry = await _queue.DequeueAsync(linkedCancellationToken.Token).AnyContext();
            } catch (Exception ex) {
                return(JobResult.FromException(ex, $"Error trying to dequeue message: {ex.Message}"));
            }

            if (queueEntry == null)
            {
                return(JobResult.Success);
            }

            if (context.CancellationToken.IsCancellationRequested)
            {
                _logger.Info("Job was cancelled. Abandoning queue item: {queueEntryId}", queueEntry.Id);
                await queueEntry.AbandonAsync().AnyContext();

                return(JobResult.Cancelled);
            }

            using (var lockValue = await GetQueueEntryLockAsync(queueEntry, context.CancellationToken).AnyContext()) {
                if (lockValue == null)
                {
                    await queueEntry.AbandonAsync().AnyContext();

                    return(JobResult.SuccessWithMessage("Unable to acquire queue item lock."));
                }

                _logger.Info("Processing {0} queue entry ({1}).", _queueEntryName, queueEntry.Id);

                try {
                    var result = await ProcessQueueEntryAsync(new JobQueueEntryContext <T>(queueEntry, lockValue, context.CancellationToken)).AnyContext();

                    if (!AutoComplete)
                    {
                        return(result);
                    }

                    if (result.IsSuccess)
                    {
                        await queueEntry.CompleteAsync().AnyContext();

                        _logger.Info("Completed {0} queue entry ({1}).", _queueEntryName, queueEntry.Id);
                    }
                    else
                    {
                        await queueEntry.AbandonAsync().AnyContext();

                        _logger.Warn("Abandoned {0} queue entry ({1}).", _queueEntryName, queueEntry.Id);
                    }

                    return(result);
                } catch (Exception ex) {
                    await queueEntry.AbandonAsync().AnyContext();

                    _logger.Error(ex, "Error processing {0} queue entry ({1}).", _queueEntryName, queueEntry.Id);

                    throw;
                }
            }
        }