protected override Task <JobResult> RunInternalAsync(CancellationToken token)
{
QueueEntry <T> queueEntry = null;
try {
queueEntry = _queue.Dequeue();
} catch (Exception ex) {
if (!(ex is TimeoutException))
{
Log.Error().Exception(ex).Message("Error trying to dequeue message: {0}", ex.Message).Write();
return(Task.FromResult(JobResult.FromException(ex)));
}
}
if (queueEntry == null)
{
return(Task.FromResult(JobResult.Success));
}
var lockValue = GetQueueItemLock(queueEntry);
if (lockValue == null)
{
Log.Warn().Message("Unable to acquire lock for queue entry '{0}'.", queueEntry.Id).Write();
return(Task.FromResult(JobResult.FailedWithMessage("Unable to acquire lock for queue entry.")));
}
Log.Trace().Message("Processing queue entry '{0}'.", queueEntry.Id).Write();
using (lockValue)
return(ProcessQueueItem(queueEntry));
}
protected async override Task <JobResult> ProcessQueueItem(QueueEntry <WorkItemData> queueEntry)
{
var workItemDataType = Type.GetType(queueEntry.Value.Type);
if (workItemDataType == null)
{
return(JobResult.FailedWithMessage("Could not resolve work item data type."));
}
object workItemData;
try {
workItemData = _queue.Serializer.Deserialize(queueEntry.Value.Data, workItemDataType);
} catch (Exception ex) {
return(JobResult.FromException(ex, "Failed to parse work item data."));
}
var handler = _handlers.GetHandler(workItemDataType);
if (handler == null)
{
return(JobResult.FailedWithMessage("Handler for type {0} not registered.", workItemDataType.Name));
}
var progressCallback = new Action <int, string>((progress, message) => _messageBus.Publish(new WorkItemStatus {
WorkItemId = queueEntry.Value.WorkItemId,
Progress = progress,
Message = message,
Type = queueEntry.Value.Type
}));
if (queueEntry.Value.SendProgressReports)
{
_messageBus.Publish(new WorkItemStatus {
WorkItemId = queueEntry.Value.WorkItemId,
Progress = 0,
Type = queueEntry.Value.Type
});
}
try {
await handler.HandleItem(new WorkItemContext(workItemData, progressCallback));
} catch (Exception ex) {
return(JobResult.FromException(ex, "Error in handler {0}.", workItemDataType.Name));
}
queueEntry.Complete();
if (queueEntry.Value.SendProgressReports)
{
_messageBus.Publish(new WorkItemStatus {
WorkItemId = queueEntry.Value.WorkItemId,
Progress = 100,
Type = queueEntry.Value.Type
});
}
return(JobResult.Success);
}
public async Task <JobResult> RunAsync(Type jobType, TimeSpan?initialDelay = null, CancellationToken cancellationToken = default(CancellationToken))
{
if (initialDelay.HasValue && initialDelay.Value > TimeSpan.Zero)
{
await Task.Delay(initialDelay.Value, cancellationToken).AnyContext();
}
var jobInstance = CreateJobInstance(jobType);
if (jobInstance == null)
{
return(JobResult.FailedWithMessage("Could not create job instance: " + jobType));
}
return(await jobInstance.RunAsync(cancellationToken).AnyContext());
}
public Task <JobResult> RunAsync(CancellationToken cancellationToken = default(CancellationToken))
{
try {
var lockValue = GetJobLock();
if (lockValue == null)
{
Log.Warn().Message("Unable to acquire job lock").Write();
return(Task.FromResult(JobResult.FailedWithMessage("Unable to acquire job lock.")));
}
using (lockValue)
return(TryRunAsync(cancellationToken));
} catch (TimeoutException) {
return(Task.FromResult(JobResult.FailedWithMessage("Timeout attempting to acquire lock.")));
}
}
public async Task <JobResult> RunAsync(CancellationToken cancellationToken = default(CancellationToken))
{
var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, TimeSpan.FromSeconds(30).ToCancellationToken());
IQueueEntry <WorkItemData> queueEntry;
try {
queueEntry = await _queue.DequeueAsync(linkedCancellationTokenSource.Token).AnyContext();
} catch (Exception ex) {
return(JobResult.FromException(ex, $"Error trying to dequeue work item: {ex.Message}"));
}
if (queueEntry == null)
{
return(JobResult.Success);
}
if (cancellationToken.IsCancellationRequested)
{
_logger.Info("Job was cancelled. Abandoning {queueEntryType} work item: {queueEntryId}", queueEntry.Value.Type, queueEntry.Id);
await queueEntry.AbandonAsync().AnyContext();
return(JobResult.CancelledWithMessage($"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}"));
}
Type workItemDataType = null;
try {
workItemDataType = Type.GetType(queueEntry.Value.Type);
} catch (Exception ex) {
await queueEntry.AbandonAsync().AnyContext();
return(JobResult.FromException(ex, $"Could not resolve {workItemDataType.Name} work item data type."));
}
if (workItemDataType == null)
{
await queueEntry.AbandonAsync().AnyContext();
return(JobResult.FailedWithMessage($"Could not resolve {workItemDataType.Name} work item data type."));
}
object workItemData;
try {
workItemData = await _queue.Serializer.DeserializeAsync(queueEntry.Value.Data, workItemDataType).AnyContext();
} catch (Exception ex) {
await queueEntry.AbandonAsync().AnyContext();
return(JobResult.FromException(ex, $"Failed to parse {workItemDataType.Name} work item data."));
}
var handler = _handlers.GetHandler(workItemDataType);
if (handler == null)
{
await queueEntry.CompleteAsync().AnyContext();
return(JobResult.FailedWithMessage($"Handler for type {workItemDataType.Name} not registered."));
}
if (queueEntry.Value.SendProgressReports)
{
await _messageBus.PublishAsync(new WorkItemStatus {
WorkItemId = queueEntry.Value.WorkItemId,
Progress = 0,
Type = queueEntry.Value.Type
}).AnyContext();
}
var lockValue = await handler.GetWorkItemLockAsync(workItemData, cancellationToken).AnyContext();
if (lockValue == null)
{
await queueEntry.AbandonAsync().AnyContext();
handler.Log.Trace("Unable to acquire work item lock.");
return(JobResult.Success);
}
var progressCallback = new Func <int, string, Task>(async(progress, message) => {
if (handler.AutoRenewLockOnProgress)
{
await queueEntry.RenewLockAsync().AnyContext();
}
if (handler.AutoRenewLockOnProgress)
{
await lockValue.RenewAsync().AnyContext();
}
await _messageBus.PublishAsync(new WorkItemStatus {
WorkItemId = queueEntry.Value.WorkItemId,
Progress = progress,
Message = message,
Type = queueEntry.Value.Type
}).AnyContext();
});
try {
handler.LogProcessingQueueEntry(queueEntry, workItemDataType, workItemData);
await handler.HandleItemAsync(new WorkItemContext(workItemData, JobId, lockValue, cancellationToken, progressCallback)).AnyContext();
if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted)
{
await queueEntry.CompleteAsync().AnyContext();
handler.LogAutoCompletedQueueEntry(queueEntry, workItemDataType, workItemData);
}
if (queueEntry.Value.SendProgressReports)
{
await _messageBus.PublishAsync(new WorkItemStatus {
WorkItemId = queueEntry.Value.WorkItemId,
Progress = 100,
Type = queueEntry.Value.Type
}).AnyContext();
}
return(JobResult.Success);
} catch (Exception ex) {
if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted)
{
await queueEntry.AbandonAsync().AnyContext();
}
handler.Log.Error(ex, "Error processing {0} work item queue entry ({1}).", workItemDataType.Name, queueEntry.Id);
return(JobResult.FromException(ex, $"Error in handler {workItemDataType.Name}."));
} finally {
await lockValue.ReleaseAsync().AnyContext();
}
}
public async Task <JobResult> ProcessAsync(IQueueEntry <WorkItemData> queueEntry, CancellationToken cancellationToken)
{
if (queueEntry == null)
{
return(JobResult.Success);
}
if (cancellationToken.IsCancellationRequested)
{
await queueEntry.AbandonAsync().AnyContext();
return(JobResult.CancelledWithMessage($"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}"));
}
Type workItemDataType;
try {
workItemDataType = Type.GetType(queueEntry.Value.Type);
} catch (Exception ex) {
await queueEntry.AbandonAsync().AnyContext();
return(JobResult.FromException(ex, $"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}: Could not resolve work item data type."));
}
if (workItemDataType == null)
{
await queueEntry.AbandonAsync().AnyContext();
return(JobResult.FailedWithMessage($"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}: Could not resolve work item data type."));
}
object workItemData;
try {
workItemData = _queue.Serializer.Deserialize(queueEntry.Value.Data, workItemDataType);
} catch (Exception ex) {
await queueEntry.AbandonAsync().AnyContext();
return(JobResult.FromException(ex, $"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}: Failed to parse {workItemDataType.Name} work item data."));
}
var handler = _handlers.GetHandler(workItemDataType);
if (handler == null)
{
await queueEntry.CompleteAsync().AnyContext();
return(JobResult.FailedWithMessage($"Completing {queueEntry.Value.Type} work item: {queueEntry.Id}: Handler for type {workItemDataType.Name} not registered."));
}
if (queueEntry.Value.SendProgressReports)
{
await ReportProgressAsync(handler, queueEntry).AnyContext();
}
var lockValue = await handler.GetWorkItemLockAsync(workItemData, cancellationToken).AnyContext();
if (lockValue == null)
{
if (handler.Log.IsEnabled(LogLevel.Information))
{
handler.Log.LogInformation("Abandoning {TypeName} work item: {Id}: Unable to acquire work item lock.", queueEntry.Value.Type, queueEntry.Id);
}
await queueEntry.AbandonAsync().AnyContext();
return(JobResult.Success);
}
var progressCallback = new Func <int, string, Task>(async(progress, message) => {
if (handler.AutoRenewLockOnProgress)
{
try {
await Task.WhenAll(
queueEntry.RenewLockAsync(),
lockValue.RenewAsync()
).AnyContext();
} catch (Exception ex) {
if (handler.Log.IsEnabled(LogLevel.Error))
{
handler.Log.LogError(ex, "Error renewing work item locks: {Message}", ex.Message);
}
}
}
await ReportProgressAsync(handler, queueEntry, progress, message).AnyContext();
if (handler.Log.IsEnabled(LogLevel.Information))
{
handler.Log.LogInformation("{TypeName} Progress {Progress}%: {Message}", workItemDataType.Name, progress, message);
}
});
try {
handler.LogProcessingQueueEntry(queueEntry, workItemDataType, workItemData);
await handler.HandleItemAsync(new WorkItemContext(workItemData, JobId, lockValue, cancellationToken, progressCallback)).AnyContext();
if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted)
{
await queueEntry.CompleteAsync().AnyContext();
handler.LogAutoCompletedQueueEntry(queueEntry, workItemDataType, workItemData);
}
if (queueEntry.Value.SendProgressReports)
{
await ReportProgressAsync(handler, queueEntry, 100).AnyContext();
}
return(JobResult.Success);
} catch (Exception ex) {
if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted)
{
await queueEntry.AbandonAsync().AnyContext();
return(JobResult.FromException(ex, $"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}: Error in handler {workItemDataType.Name}."));
}
return(JobResult.FromException(ex, $"Error processing {queueEntry.Value.Type} work item: {queueEntry.Id} in handler: {workItemDataType.Name}"));
} finally {
await lockValue.ReleaseAsync().AnyContext();
}
}
protected async override Task <JobResult> ProcessQueueEntryAsync(JobQueueEntryContext <WorkItemData> context)
{
var workItemDataType = Type.GetType(context.QueueEntry.Value.Type);
if (workItemDataType == null)
{
return(JobResult.FailedWithMessage("Could not resolve work item data type."));
}
object workItemData;
try {
workItemData = await _queue.Serializer.DeserializeAsync(context.QueueEntry.Value.Data, workItemDataType).AnyContext();
} catch (Exception ex) {
return(JobResult.FromException(ex, "Failed to parse work item data."));
}
var handler = _handlers.GetHandler(workItemDataType);
if (handler == null)
{
return(JobResult.FailedWithMessage($"Handler for type {workItemDataType.Name} not registered."));
}
if (context.QueueEntry.Value.SendProgressReports)
{
await _messageBus.PublishAsync(new WorkItemStatus {
WorkItemId = context.QueueEntry.Value.WorkItemId,
Progress = 0,
Type = context.QueueEntry.Value.Type
}).AnyContext();
}
using (var lockValue = await handler.GetWorkItemLockAsync(workItemData, context.CancellationToken).AnyContext()) {
if (lockValue == null)
{
return(JobResult.SuccessWithMessage("Unable to acquire work item lock."));
}
var progressCallback = new Func <int, string, Task>(async(progress, message) => {
if (handler.AutoRenewLockOnProgress && lockValue != null)
{
await lockValue.RenewAsync().AnyContext();
}
await _messageBus.PublishAsync(new WorkItemStatus {
WorkItemId = context.QueueEntry.Value.WorkItemId,
Progress = progress,
Message = message,
Type = context.QueueEntry.Value.Type
}).AnyContext();
});
try {
_logger.Info("Processing {0} work item queue entry ({1}).", workItemDataType.Name, context.QueueEntry.Id);
await handler.HandleItemAsync(new WorkItemContext(context, workItemData, JobId, lockValue, progressCallback)).AnyContext();
} catch (Exception ex) {
await context.QueueEntry.AbandonAsync().AnyContext();
_logger.Error("Error processing {0} work item queue entry ({1}).", workItemDataType.Name, context.QueueEntry.Id);
return(JobResult.FromException(ex, $"Error in handler {workItemDataType.Name}."));
}
await context.QueueEntry.CompleteAsync().AnyContext();
_logger.Info("Completed {0} work item queue entry ({1}).", workItemDataType.Name, context.QueueEntry.Id);
if (context.QueueEntry.Value.SendProgressReports)
{
await _messageBus.PublishAsync(new WorkItemStatus {
WorkItemId = context.QueueEntry.Value.WorkItemId,
Progress = 100,
Type = context.QueueEntry.Value.Type
}).AnyContext();
}
return(JobResult.Success);
}
}