public async Task <JobResult> ProcessAsync(IQueueEntry <T> queueEntry, CancellationToken cancellationToken) { if (queueEntry == null) { return(JobResult.Success); } if (cancellationToken.IsCancellationRequested) { _logger.Info(() => $"Job was cancelled. Abandoning {_queueEntryName} queue item: {queueEntry.Id}"); await queueEntry.AbandonAsync().AnyContext(); return(JobResult.CancelledWithMessage($"Abandoning {_queueEntryName} queue item: {queueEntry.Id}")); } var lockValue = await GetQueueEntryLockAsync(queueEntry, cancellationToken).AnyContext(); if (lockValue == null) { await queueEntry.AbandonAsync().AnyContext(); _logger.Trace("Unable to acquire queue entry lock."); return(JobResult.Success); } try { LogProcessingQueueEntry(queueEntry); var result = await ProcessQueueEntryAsync(new QueueEntryContext <T>(queueEntry, lockValue, cancellationToken)).AnyContext(); if (!AutoComplete || queueEntry.IsCompleted || queueEntry.IsAbandoned) { return(result); } if (result.IsSuccess) { await queueEntry.CompleteAsync().AnyContext(); LogAutoCompletedQueueEntry(queueEntry); } else { await queueEntry.AbandonAsync().AnyContext(); _logger.Warn(() => $"Auto abandoned {_queueEntryName} queue entry ({queueEntry.Id})."); } return(result); } catch (Exception ex) { _logger.Error(ex, () => $"Error processing {_queueEntryName} queue entry ({queueEntry.Id})."); if (!queueEntry.IsCompleted && !queueEntry.IsAbandoned) { await queueEntry.AbandonAsync().AnyContext(); } throw; } finally { await lockValue.ReleaseAsync().AnyContext(); } }
protected override void StartWorkingImpl(Func <IQueueEntry <T>, CancellationToken, Task> handler, bool autoComplete, CancellationToken cancellationToken) { if (handler == null) { throw new ArgumentNullException(nameof(handler)); } var linkedCancellationToken = GetLinkedDisposableCanncellationTokenSource(cancellationToken); Task.Run(async() => { bool isTraceLogLevelEnabled = _logger.IsEnabled(LogLevel.Trace); if (isTraceLogLevelEnabled) { _logger.LogTrace("WorkerLoop Start {Name}", _options.Name); } while (!linkedCancellationToken.IsCancellationRequested) { if (isTraceLogLevelEnabled) { _logger.LogTrace("WorkerLoop Signaled {Name}", _options.Name); } IQueueEntry <T> queueEntry = null; try { queueEntry = await DequeueImplAsync(linkedCancellationToken.Token).AnyContext(); } catch (OperationCanceledException) { } if (linkedCancellationToken.IsCancellationRequested || queueEntry == null) { continue; } try { await handler(queueEntry, linkedCancellationToken.Token).AnyContext(); if (autoComplete && !queueEntry.IsAbandoned && !queueEntry.IsCompleted) { await queueEntry.CompleteAsync().AnyContext(); } } catch (Exception ex) { Interlocked.Increment(ref _workerErrorCount); if (_logger.IsEnabled(LogLevel.Error)) { _logger.LogError(ex, "Worker error: {Message}", ex.Message); } if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted) { await queueEntry.AbandonAsync().AnyContext(); } } } if (isTraceLogLevelEnabled) { _logger.LogTrace("Worker exiting: {Name} Cancel Requested: {IsCancellationRequested}", _queueReference.Name, linkedCancellationToken.IsCancellationRequested); } }, linkedCancellationToken.Token).ContinueWith(t => linkedCancellationToken.Dispose()); }
protected async Task DoWorkAsync(IQueueEntry <SimpleWorkItem> w, AsyncCountdownEvent countdown, WorkInfo info) { _logger.Trace($"Starting: {w.Value.Id}"); Assert.Equal("Hello", w.Value.Data); try { // randomly complete, abandon or blowup. if (RandomData.GetBool()) { _logger.Trace($"Completing: {w.Value.Id}"); await w.CompleteAsync(); info.IncrementCompletedCount(); } else if (RandomData.GetBool()) { _logger.Trace($"Abandoning: {w.Value.Id}"); await w.AbandonAsync(); info.IncrementAbandonCount(); } else { _logger.Trace($"Erroring: {w.Value.Id}"); info.IncrementErrorCount(); throw new Exception(); } } finally { _logger.Trace($"Signal {countdown.CurrentCount}"); countdown.Signal(); } }
protected override void StartWorkingImpl(Func <IQueueEntry <T>, CancellationToken, Task> handler, bool autoComplete, CancellationToken cancellationToken) { if (handler == null) { throw new ArgumentNullException(nameof(handler)); } _logger.LogTrace("Queue {Name} start working", _options.Name); _workers.Add(Task.Run(async() => { using var linkedCancellationToken = GetLinkedDisposableCancellationTokenSource(cancellationToken); _logger.LogTrace("WorkerLoop Start {Name}", _options.Name); while (!linkedCancellationToken.IsCancellationRequested) { _logger.LogTrace("WorkerLoop Signaled {Name}", _options.Name); IQueueEntry <T> queueEntry = null; try { queueEntry = await DequeueImplAsync(linkedCancellationToken.Token).AnyContext(); } catch (Exception ex) { _logger.LogError(ex, "Error on Dequeue: {Message}", ex.Message); } if (linkedCancellationToken.IsCancellationRequested || queueEntry == null) { return; } try { await handler(queueEntry, linkedCancellationToken.Token).AnyContext(); } catch (Exception ex) { _logger.LogError(ex, "Worker error: {Message}", ex.Message); if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted) { try { await queueEntry.AbandonAsync().AnyContext(); } catch (Exception abandonEx) { _logger.LogError(abandonEx, "Worker error abandoning queue entry: {Message}", abandonEx.Message); } } Interlocked.Increment(ref _workerErrorCount); } if (autoComplete && !queueEntry.IsAbandoned && !queueEntry.IsCompleted) { try { await Run.WithRetriesAsync(() => queueEntry.CompleteAsync(), cancellationToken: linkedCancellationToken.Token, logger: _logger).AnyContext(); } catch (Exception ex) { _logger.LogError(ex, "Worker error attempting to auto complete entry: {Message}", ex.Message); } } } _logger.LogTrace("Worker exiting: {Name} Cancel Requested: {IsCancellationRequested}", _options.Name, linkedCancellationToken.IsCancellationRequested); }, GetLinkedDisposableCancellationTokenSource(cancellationToken).Token)); }
public override void StartWorking(Func <IQueueEntry <T>, CancellationToken, Task> handler, bool autoComplete = false, CancellationToken cancellationToken = default(CancellationToken)) { if (handler == null) { throw new ArgumentNullException(nameof(handler)); } _logger.Trace("Queue {0} start working", typeof(T).Name); var linkedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(_disposeTokenSource.Token, cancellationToken).Token; Task.Run(async() => { _logger.Trace("WorkerLoop Start {0}", typeof(T).Name); while (!linkedCancellationToken.IsCancellationRequested) { _logger.Trace("WorkerLoop Signaled {0}", typeof(T).Name); IQueueEntry <T> queueEntry = null; try { queueEntry = await DequeueAsync(cancellationToken: cancellationToken).AnyContext(); } catch (Exception ex) { _logger.Error(ex, "Error on Dequeue: " + ex.Message); } if (queueEntry == null) { return; } try { await handler(queueEntry, linkedCancellationToken).AnyContext(); if (autoComplete && !queueEntry.IsAbandoned && !queueEntry.IsCompleted) { await queueEntry.CompleteAsync().AnyContext(); } } catch (Exception ex) { _logger.Error(ex, "Worker error: {0}", ex.Message); if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted) { await queueEntry.AbandonAsync().AnyContext(); } Interlocked.Increment(ref _workerErrorCount); } } _logger.Trace("WorkLoop End"); }, linkedCancellationToken); }
protected override void StartWorkingImpl(Func <IQueueEntry <T>, CancellationToken, Task> handler, bool autoComplete, CancellationToken cancellationToken) { if (handler == null) { throw new ArgumentNullException(nameof(handler)); } _logger.Trace("Queue {0} start working", _options.Name); var linkedCancellationToken = GetLinkedDisposableCanncellationToken(cancellationToken); Task.Run(async() => { _logger.Trace("WorkerLoop Start {0}", _options.Name); while (!linkedCancellationToken.IsCancellationRequested) { _logger.Trace("WorkerLoop Signaled {0}", _options.Name); IQueueEntry <T> queueEntry = null; try { queueEntry = await DequeueImplAsync(linkedCancellationToken).AnyContext(); } catch (Exception ex) { _logger.Error(ex, "Error on Dequeue: " + ex.Message); } if (linkedCancellationToken.IsCancellationRequested || queueEntry == null) { return; } try { await handler(queueEntry, linkedCancellationToken).AnyContext(); if (autoComplete && !queueEntry.IsAbandoned && !queueEntry.IsCompleted) { await queueEntry.CompleteAsync().AnyContext(); } } catch (Exception ex) { _logger.Error(ex, "Worker error: {0}", ex.Message); if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted) { await queueEntry.AbandonAsync().AnyContext(); } Interlocked.Increment(ref _workerErrorCount); } } _logger.Trace("Worker exiting: {0} Cancel Requested: {1}", _options.Name, linkedCancellationToken.IsCancellationRequested); }, linkedCancellationToken); }
protected override void StartWorkingImpl(Func <IQueueEntry <T>, CancellationToken, Task> handler, bool autoComplete, CancellationToken cancellationToken) { if (handler == null) { throw new ArgumentNullException(nameof(handler)); } var linkedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(_queueDisposedCancellationTokenSource.Token, cancellationToken).Token; Task.Run(async() => { _logger.Trace("WorkerLoop Start {_queueName}", _queueName); while (!linkedCancellationToken.IsCancellationRequested) { _logger.Trace("WorkerLoop Signaled {_queueName}", _queueName); IQueueEntry <T> queueEntry = null; try { queueEntry = await DequeueImplAsync(cancellationToken: cancellationToken).AnyContext(); } catch (TimeoutException) { } if (linkedCancellationToken.IsCancellationRequested || queueEntry == null) { continue; } try { await handler(queueEntry, linkedCancellationToken).AnyContext(); if (autoComplete && !queueEntry.IsAbandoned && !queueEntry.IsCompleted) { await queueEntry.CompleteAsync().AnyContext(); } } catch (Exception ex) { Interlocked.Increment(ref _workerErrorCount); _logger.Error(ex, "Worker error: {0}", ex.Message); if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted) { await queueEntry.AbandonAsync().AnyContext(); } } } _logger.Trace("Worker exiting: {0} Cancel Requested: {1}", _queueName, linkedCancellationToken.IsCancellationRequested); }, linkedCancellationToken); }
public override void StartWorking(Func <IQueueEntry <T>, CancellationToken, Task> handler, bool autoComplete = false, CancellationToken cancellationToken = new CancellationToken()) { if (handler == null) { throw new ArgumentNullException(nameof(handler)); } var linkedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(_queueDisposedCancellationTokenSource.Token, cancellationToken).Token; Task.Run(async() => { while (!linkedCancellationToken.IsCancellationRequested) { IQueueEntry <T> queueEntry = null; try { queueEntry = await DequeueAsync(cancellationToken).AnyContext(); } catch (TaskCanceledException) { } if (linkedCancellationToken.IsCancellationRequested || queueEntry == null) { continue; } try { await handler(queueEntry, cancellationToken); if (autoComplete) { await queueEntry.CompleteAsync().AnyContext(); } } catch (Exception ex) { _logger.Error(ex, "Worker error: {0}", ex.Message); await queueEntry.AbandonAsync().AnyContext(); Interlocked.Increment(ref _workerErrorCount); } } _logger.Trace("Worker exiting: {0} Cancel Requested: {1}", _queueReference.Name, linkedCancellationToken.IsCancellationRequested); }, linkedCancellationToken); }
public Task AbandonAsync() { return(_queueEntry.AbandonAsync()); }
public async Task <JobResult> ProcessAsync(IQueueEntry <WorkItemData> queueEntry, CancellationToken cancellationToken) { if (queueEntry == null) { return(JobResult.Success); } if (cancellationToken.IsCancellationRequested) { await queueEntry.AbandonAsync().AnyContext(); return(JobResult.CancelledWithMessage($"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}")); } Type workItemDataType; try { workItemDataType = Type.GetType(queueEntry.Value.Type); } catch (Exception ex) { await queueEntry.AbandonAsync().AnyContext(); return(JobResult.FromException(ex, $"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}: Could not resolve work item data type.")); } if (workItemDataType == null) { await queueEntry.AbandonAsync().AnyContext(); return(JobResult.FailedWithMessage($"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}: Could not resolve work item data type.")); } object workItemData; try { workItemData = _queue.Serializer.Deserialize(queueEntry.Value.Data, workItemDataType); } catch (Exception ex) { await queueEntry.AbandonAsync().AnyContext(); return(JobResult.FromException(ex, $"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}: Failed to parse {workItemDataType.Name} work item data.")); } var handler = _handlers.GetHandler(workItemDataType); if (handler == null) { await queueEntry.CompleteAsync().AnyContext(); return(JobResult.FailedWithMessage($"Completing {queueEntry.Value.Type} work item: {queueEntry.Id}: Handler for type {workItemDataType.Name} not registered.")); } if (queueEntry.Value.SendProgressReports) { await ReportProgressAsync(handler, queueEntry).AnyContext(); } var lockValue = await handler.GetWorkItemLockAsync(workItemData, cancellationToken).AnyContext(); if (lockValue == null) { if (handler.Log.IsEnabled(LogLevel.Information)) { handler.Log.LogInformation("Abandoning {TypeName} work item: {Id}: Unable to acquire work item lock.", queueEntry.Value.Type, queueEntry.Id); } await queueEntry.AbandonAsync().AnyContext(); return(JobResult.Success); } var progressCallback = new Func <int, string, Task>(async(progress, message) => { if (handler.AutoRenewLockOnProgress) { try { await Task.WhenAll( queueEntry.RenewLockAsync(), lockValue.RenewAsync() ).AnyContext(); } catch (Exception ex) { if (handler.Log.IsEnabled(LogLevel.Error)) { handler.Log.LogError(ex, "Error renewing work item locks: {Message}", ex.Message); } } } await ReportProgressAsync(handler, queueEntry, progress, message).AnyContext(); if (handler.Log.IsEnabled(LogLevel.Information)) { handler.Log.LogInformation("{TypeName} Progress {Progress}%: {Message}", workItemDataType.Name, progress, message); } }); try { handler.LogProcessingQueueEntry(queueEntry, workItemDataType, workItemData); await handler.HandleItemAsync(new WorkItemContext(workItemData, JobId, lockValue, cancellationToken, progressCallback)).AnyContext(); if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted) { await queueEntry.CompleteAsync().AnyContext(); handler.LogAutoCompletedQueueEntry(queueEntry, workItemDataType, workItemData); } if (queueEntry.Value.SendProgressReports) { await ReportProgressAsync(handler, queueEntry, 100).AnyContext(); } return(JobResult.Success); } catch (Exception ex) { if (!queueEntry.IsAbandoned && !queueEntry.IsCompleted) { await queueEntry.AbandonAsync().AnyContext(); return(JobResult.FromException(ex, $"Abandoning {queueEntry.Value.Type} work item: {queueEntry.Id}: Error in handler {workItemDataType.Name}.")); } return(JobResult.FromException(ex, $"Error processing {queueEntry.Value.Type} work item: {queueEntry.Id} in handler: {workItemDataType.Name}")); } finally { await lockValue.ReleaseAsync().AnyContext(); } }
public async Task <JobResult> ProcessAsync(IQueueEntry <T> queueEntry, CancellationToken cancellationToken) { if (queueEntry == null) { return(JobResult.Success); } using (_logger.BeginScope(s => s .Property("QueueEntryId", queueEntry.Id) .PropertyIf("CorrelationId", queueEntry.CorrelationId, !String.IsNullOrEmpty(queueEntry.CorrelationId)) .Property("QueueEntryName", _queueEntryName))) { _logger.LogInformation("Processing queue entry: id={QueueEntryId} type={QueueEntryName} attempt={QueueEntryAttempt}", queueEntry.Id, _queueEntryName, queueEntry.Attempts); if (cancellationToken.IsCancellationRequested) { if (_logger.IsEnabled(LogLevel.Information)) { _logger.LogInformation("Job was cancelled. Abandoning {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id); } await queueEntry.AbandonAsync().AnyContext(); return(JobResult.CancelledWithMessage($"Abandoning {_queueEntryName} queue entry: {queueEntry.Id}")); } var lockValue = await GetQueueEntryLockAsync(queueEntry, cancellationToken).AnyContext(); if (lockValue == null) { await queueEntry.AbandonAsync().AnyContext(); _logger.LogTrace("Unable to acquire queue entry lock."); return(JobResult.Success); } bool isTraceLogLevelEnabled = _logger.IsEnabled(LogLevel.Trace); try { LogProcessingQueueEntry(queueEntry); var result = await ProcessQueueEntryAsync(new QueueEntryContext <T>(queueEntry, lockValue, cancellationToken)).AnyContext(); if (!AutoComplete || queueEntry.IsCompleted || queueEntry.IsAbandoned) { return(result); } if (result.IsSuccess) { await queueEntry.CompleteAsync().AnyContext(); LogAutoCompletedQueueEntry(queueEntry); } else { if (result.Error != null || result.Message != null) { _logger.LogError(result.Error, "{QueueEntryName} queue entry {Id} returned an unsuccessful response: {ErrorMessage}", _queueEntryName, queueEntry.Id, result.Message ?? result.Error?.Message); } if (isTraceLogLevelEnabled) { _logger.LogTrace("Processing was not successful. Auto Abandoning {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id); } await queueEntry.AbandonAsync().AnyContext(); if (_logger.IsEnabled(LogLevel.Warning)) { _logger.LogWarning("Auto abandoned {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id); } } return(result); } catch (Exception ex) { _logger.LogError(ex, "Error processing {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id); if (!queueEntry.IsCompleted && !queueEntry.IsAbandoned) { await queueEntry.AbandonAsync().AnyContext(); } throw; } finally { if (isTraceLogLevelEnabled) { _logger.LogTrace("Releasing Lock for {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id); } await lockValue.ReleaseAsync().AnyContext(); if (isTraceLogLevelEnabled) { _logger.LogTrace("Released Lock for {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id); } } } }
public async Task <JobResult> ProcessAsync(IQueueEntry <T> queueEntry, CancellationToken cancellationToken) { if (queueEntry == null) { return(JobResult.Success); } if (cancellationToken.IsCancellationRequested) { if (_logger.IsEnabled(LogLevel.Information)) { _logger.LogInformation("Job was cancelled. Abandoning {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id); } await queueEntry.AbandonAsync().AnyContext(); return(JobResult.CancelledWithMessage($"Abandoning {_queueEntryName} queue entry: {queueEntry.Id}")); } var lockValue = await GetQueueEntryLockAsync(queueEntry, cancellationToken).AnyContext(); if (lockValue == null) { await queueEntry.AbandonAsync().AnyContext(); _logger.LogTrace("Unable to acquire queue entry lock."); return(JobResult.Success); } bool isTraceLogLevelEnabled = _logger.IsEnabled(LogLevel.Trace); try { LogProcessingQueueEntry(queueEntry); var result = await ProcessQueueEntryAsync(new QueueEntryContext <T>(queueEntry, lockValue, cancellationToken)).AnyContext(); if (!AutoComplete || queueEntry.IsCompleted || queueEntry.IsAbandoned) { return(result); } if (result.IsSuccess) { await queueEntry.CompleteAsync().AnyContext(); LogAutoCompletedQueueEntry(queueEntry); } else { if (isTraceLogLevelEnabled) { _logger.LogTrace("Processing was not successful. Auto Abandoning {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id); } await queueEntry.AbandonAsync().AnyContext(); if (_logger.IsEnabled(LogLevel.Warning)) { _logger.LogWarning("Auto abandoned {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id); } } return(result); } catch (Exception ex) { if (_logger.IsEnabled(LogLevel.Error)) { _logger.LogError(ex, "Error processing {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id); } if (!queueEntry.IsCompleted && !queueEntry.IsAbandoned) { await queueEntry.AbandonAsync().AnyContext(); } throw; } finally { if (isTraceLogLevelEnabled) { _logger.LogTrace("Releasing Lock for {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id); } await lockValue.ReleaseAsync().AnyContext(); if (isTraceLogLevelEnabled) { _logger.LogTrace("Released Lock for {QueueEntryName} queue entry: {Id}", _queueEntryName, queueEntry.Id); } } }
private async Task AbandonEntryAsync(IQueueEntry <EventPost> queueEntry) { await queueEntry.AbandonAsync().AnyContext(); await _storage.SetNotActiveAsync(queueEntry.Value.FilePath, _logger).AnyContext(); }
private static async Task DoAbandonMessageAsync(IQueueEntry <TMessage> queueEntry) { await queueEntry.AbandonAsync() .ConfigureAwait(false); }
protected async Task DoWorkAsync(IQueueEntry<SimpleWorkItem> w, AsyncCountdownEvent countdown, WorkInfo info) { _logger.Trace($"Starting: {w.Value.Id}"); Assert.Equal("Hello", w.Value.Data); try { // randomly complete, abandon or blowup. if (RandomData.GetBool()) { _logger.Trace($"Completing: {w.Value.Id}"); await w.CompleteAsync(); info.IncrementCompletedCount(); } else if (RandomData.GetBool()) { _logger.Trace($"Abandoning: {w.Value.Id}"); await w.AbandonAsync(); info.IncrementAbandonCount(); } else { _logger.Trace($"Erroring: {w.Value.Id}"); info.IncrementErrorCount(); throw new Exception(); } } finally { _logger.Trace($"Signal {countdown.CurrentCount}"); countdown.Signal(); } }