private async Task <Exception> ThrowBpmnError <TCamundaWorker>(IOperation operation, IOperationLogger logger, LockedExternalTaskListItem item, TCamundaWorker worker, CamundaWorkerBpmnError error) where TCamundaWorker : CamundaWorker { try { var retries = Properties.CountOfRetriesWhenFailuresAre; var retryAfter = TimeSpan.FromMilliseconds(Properties.RetryAfterFailureTimeout); ExternalTask.HandleBpmnErrorResponse response = null; while (retries > 0) { response = await new ExternalTask.HandleBpmnErrorRequest(item.Id) { WorkerId = item.WorkerId, ErrorCode = error.Code, ErrorMessage = error.Message, Variables = worker.CollectVariables() }.SendRequest(operation, CamundaClient); if (!response.IsWithoutErrors()) { retries--; await logger.Error($"[Operation: {operation.ComplexOperationId}] Couldn't send BPMN error for task, retry task handling after {retryAfter}... (remaining attempts: {retries})", response.ToLog()); if (retries != 0) { await Task.Delay(retryAfter); } } else { await logger.Debug($"[Operation: {operation.ComplexOperationId}] Send BPMN error is successful"); break; } } if (retries == 0) { await logger.Fatal($"[Operation: {operation.ComplexOperationId}] Couldn't send BPMN error for task, stop retrying"); return(new Exception(response?.Output)); } } catch (Exception e) { await logger.Error($"[Operation: {operation.ComplexOperationId}] Failed to send BPMN error for task", e.ToLog()); return(e); } return(null); }
private async Task Run <TCamundaWorker>(IOperation operation, IOperationLogger logger, LockedExternalTaskListItem item) where TCamundaWorker : CamundaWorker { using var scope = ServiceCollection.BuildServiceProvider(false).CreateScope(); var worker = scope.ServiceProvider.GetRequiredService <TCamundaWorker>(); worker.InitializeVariables(item.Variables); Exception exception = null; using (var cancellationTokenSource = new CancellationTokenSource()) { try { var token = cancellationTokenSource.Token; #pragma warning disable 4014 Task.Run(async() => await ExtendLock(operation, logger, item, token), token); #pragma warning restore 4014 worker.Execute(operation, logger); } catch (CamundaWorkerExecutionIsNotCompletedYet) { await logger.Debug($"[Operation: {operation.ComplexOperationId}] Skipping task"); return; } catch (CamundaWorkerBpmnError error) { await logger.Debug($"[Operation: {operation.ComplexOperationId}] BPMN error is thrown", error.ToLog()); exception = await ThrowBpmnError(operation, logger, item, worker, error); if (exception.IsEmpty()) { return; } } catch (Exception e) { await logger.Error($"[Operation: {operation.ComplexOperationId}] Failed to perform task", e.ToLog()); exception = e; } finally { cancellationTokenSource.Cancel(true); } } var retries = Properties.CountOfRetriesWhenFailuresAre; while (retries > 0) { var retryAfter = TimeSpan.FromMilliseconds(Properties.RetryAfterFailureTimeout); IntermediateCamundaResponse <EmptyResult> response; if (exception.IsEmpty()) { await logger.Trace($"[Operation: {operation.ComplexOperationId}] Trying to complete task"); response = await new ExternalTask.CompleteRequest(item.Id) { WorkerId = item.WorkerId, Variables = worker.CollectVariables() }.SendRequest(operation, CamundaClient); } else { var countOfRetries = (item.Retries ?? Properties.CountOfRetriesWhenFailuresAre) - 1; await logger.Trace($"[Operation: {operation.ComplexOperationId}] Trying to handle failure, retry task execution after {retryAfter}... (remaining attempts: {countOfRetries})"); response = await new ExternalTask.HandleFailureRequest(item.Id) { WorkerId = item.WorkerId, ErrorMessage = exception?.Message, ErrorDetails = exception?.StackTrace, Retries = countOfRetries, RetryTimeout = Properties.RetryAfterFailureTimeout }.SendRequest(operation, CamundaClient); } if (!response.IsWithoutErrors()) { retries--; var resultType = exception.IsEmpty() ? "complete task" : "handle failure"; await logger.Error($"[Operation: {operation.ComplexOperationId}] Couldn't {resultType}, retry task handling after {retryAfter}... (remaining attempts: {retries})", response.ToLog()); if (retries != 0) { await Task.Delay(retryAfter); } } else { await logger.Debug($"[Operation: {operation.ComplexOperationId}] {(exception.IsEmpty() ? "Complete task" : "Handle failure")} process is successful"); break; } } if (retries == 0) { await logger.Fatal($"[Operation: {operation.ComplexOperationId}] Couldn't {(exception.IsEmpty() ? "complete task" : "handle failure")}, stop retrying"); } if (!exception.IsEmpty()) { throw exception !; } }