private async Task ExtendLock(IOperation operation, IOperationLogger logger, LockedExternalTaskListItem item, CancellationToken token)
{
while (true)
{
try
{
while (true)
{
await logger.Debug($"[Operation: {operation.ComplexOperationId}] Extending lock");
var response = await new ExternalTask.ExtendLockRequest(item.Id)
{
WorkerId = item.WorkerId,
NewDuration = Properties.LockDuration
}.SendRequest(operation, CamundaClient);
if (!response.IsWithoutErrors())
{
var retryAfter = TimeSpan.FromSeconds(5);
await logger.Error($"[Operation: {operation.ComplexOperationId}] Couldn't extend lock for task, retry after {retryAfter}", response.ToLog());
await Task.Delay(retryAfter, token);
}
else
{
await logger.Debug($"[Operation: {operation.ComplexOperationId}] Extend lock process is successful");
await Task.Delay(TimeSpan.FromMilliseconds(Properties.LockDuration).Divide(2), token);
break;
}
}
}
catch
{
break;
}
}
}
private async Task <Exception> ThrowBpmnError <TCamundaWorker>(IOperation operation, IOperationLogger logger, LockedExternalTaskListItem item, TCamundaWorker worker,
CamundaWorkerBpmnError error) where TCamundaWorker : CamundaWorker
{
try
{
var retries = Properties.CountOfRetriesWhenFailuresAre;
var retryAfter = TimeSpan.FromMilliseconds(Properties.RetryAfterFailureTimeout);
ExternalTask.HandleBpmnErrorResponse response = null;
while (retries > 0)
{
response = await new ExternalTask.HandleBpmnErrorRequest(item.Id)
{
WorkerId = item.WorkerId,
ErrorCode = error.Code,
ErrorMessage = error.Message,
Variables = worker.CollectVariables()
}.SendRequest(operation, CamundaClient);
if (!response.IsWithoutErrors())
{
retries--;
await logger.Error($"[Operation: {operation.ComplexOperationId}] Couldn't send BPMN error for task, retry task handling after {retryAfter}... (remaining attempts: {retries})", response.ToLog());
if (retries != 0)
{
await Task.Delay(retryAfter);
}
}
else
{
await logger.Debug($"[Operation: {operation.ComplexOperationId}] Send BPMN error is successful");
break;
}
}
if (retries == 0)
{
await logger.Fatal($"[Operation: {operation.ComplexOperationId}] Couldn't send BPMN error for task, stop retrying");
return(new Exception(response?.Output));
}
}
catch (Exception e)
{
await logger.Error($"[Operation: {operation.ComplexOperationId}] Failed to send BPMN error for task", e.ToLog());
return(e);
}
return(null);
}
private async Task Run <TCamundaWorker>(IOperation operation, IOperationLogger logger, LockedExternalTaskListItem item) where TCamundaWorker : CamundaWorker
{
using var scope = ServiceCollection.BuildServiceProvider(false).CreateScope();
var worker = scope.ServiceProvider.GetRequiredService <TCamundaWorker>();
worker.InitializeVariables(item.Variables);
Exception exception = null;
using (var cancellationTokenSource = new CancellationTokenSource())
{
try
{
var token = cancellationTokenSource.Token;
#pragma warning disable 4014
Task.Run(async() => await ExtendLock(operation, logger, item, token), token);
#pragma warning restore 4014
worker.Execute(operation, logger);
}
catch (CamundaWorkerExecutionIsNotCompletedYet)
{
await logger.Debug($"[Operation: {operation.ComplexOperationId}] Skipping task");
return;
}
catch (CamundaWorkerBpmnError error)
{
await logger.Debug($"[Operation: {operation.ComplexOperationId}] BPMN error is thrown", error.ToLog());
exception = await ThrowBpmnError(operation, logger, item, worker, error);
if (exception.IsEmpty())
{
return;
}
}
catch (Exception e)
{
await logger.Error($"[Operation: {operation.ComplexOperationId}] Failed to perform task", e.ToLog());
exception = e;
}
finally
{
cancellationTokenSource.Cancel(true);
}
}
var retries = Properties.CountOfRetriesWhenFailuresAre;
while (retries > 0)
{
var retryAfter = TimeSpan.FromMilliseconds(Properties.RetryAfterFailureTimeout);
IntermediateCamundaResponse <EmptyResult> response;
if (exception.IsEmpty())
{
await logger.Trace($"[Operation: {operation.ComplexOperationId}] Trying to complete task");
response = await new ExternalTask.CompleteRequest(item.Id)
{
WorkerId = item.WorkerId,
Variables = worker.CollectVariables()
}.SendRequest(operation, CamundaClient);
}
else
{
var countOfRetries = (item.Retries ?? Properties.CountOfRetriesWhenFailuresAre) - 1;
await logger.Trace($"[Operation: {operation.ComplexOperationId}] Trying to handle failure, retry task execution after {retryAfter}... (remaining attempts: {countOfRetries})");
response = await new ExternalTask.HandleFailureRequest(item.Id)
{
WorkerId = item.WorkerId,
ErrorMessage = exception?.Message,
ErrorDetails = exception?.StackTrace,
Retries = countOfRetries,
RetryTimeout = Properties.RetryAfterFailureTimeout
}.SendRequest(operation, CamundaClient);
}
if (!response.IsWithoutErrors())
{
retries--;
var resultType = exception.IsEmpty() ? "complete task" : "handle failure";
await logger.Error($"[Operation: {operation.ComplexOperationId}] Couldn't {resultType}, retry task handling after {retryAfter}... (remaining attempts: {retries})", response.ToLog());
if (retries != 0)
{
await Task.Delay(retryAfter);
}
}
else
{
await logger.Debug($"[Operation: {operation.ComplexOperationId}] {(exception.IsEmpty() ? "Complete task" : "Handle failure")} process is successful");
break;
}
}
if (retries == 0)
{
await logger.Fatal($"[Operation: {operation.ComplexOperationId}] Couldn't {(exception.IsEmpty() ? "complete task" : "handle failure")}, stop retrying");
}
if (!exception.IsEmpty())
{
throw exception !;
}
}
