/// <summary> /// When a task fails, follow the error outflow if one exists else fail the workflow /// </summary> /// <param name="db"></param> /// <param name="workflow"></param> /// <param name="execution"></param> /// <param name="history"></param> /// <param name="decisions"></param> private static void ProcessActivityTaskFailedEvent(Database db, WorkflowObj workflow, Execution execution, History history, List <History> decisions) { // There should(!) be no contention for the data modified in this process var evt = ActivityTaskFailedEvent.Create(history); var se = ActivityTaskScheduledEvent.Create(db.GetHistoryEvent(execution.ExecutionId, evt.SchedulingEventId)); var failedTask = workflow.Tasks.Single(t => t.TaskId == se.TaskId); var nextTaskId = failedTask.FailOutflow?.Target; if (!string.IsNullOrEmpty(nextTaskId)) { var nextTask = workflow.Tasks.Single(t => t.TaskId == nextTaskId); CreateTaskScheduledEvent(db, execution, nextTask, decisions); } decisions.Add(new WorkflowExecutionFailedEvent { Reason = $"Task {se.TaskId} failed with no recovery action defined" }); AttemptCleanup(db, execution, workflow, decisions); }
/// <summary> /// Handle response for a task including heartbeats /// The returned status property identifies the type of response /// </summary> /// <param name="taskToken"></param> /// <param name="json"></param> public string ProcessTaskResponse(Guid taskToken, string json) { try { var taskResponse = JsonConvert.DeserializeObject <ActivityTaskResponse>(json); // Note that the task may have been deleted by the ActivityTaskTimeoutChecker var task = TaskLists.SingleOrDefault(q => q.TaskToken == taskToken); string response = null; int i; for (i = 0; i < RETRIES; i++) { switch (taskResponse.Status.ToLower()) { case "success": if (task == null) { log.Info($"Ignoring success response for deleted task with token: {taskToken}"); break; } var result = taskResponse.Result; var atce = new ActivityTaskCompletedEvent { Result = result, SchedulingEventId = task.TaskScheduledEventId, }; InsertHistory(this, task.Execution, atce); task.Execution.AwaitingDecision = true; TaskLists.DeleteOnSubmit(task); break; case "failure": if (task == null) { log.Info($"Ignoring failure response for deleted task with token: {taskToken}"); break; } var atfe = new ActivityTaskFailedEvent { Reason = taskResponse.Reason, Details = taskResponse.Details, SchedulingEventId = task.TaskScheduledEventId }; InsertHistory(this, task.Execution, atfe); task.Execution.AwaitingDecision = true; TaskLists.DeleteOnSubmit(task); break; case "cancelled": if (task == null) { log.Info($"Ignoring cancellation response for deleted task with token: {taskToken}"); break; } var atxe = new ActivityTaskCancelledEvent { SchedulingEventId = task.TaskScheduledEventId }; InsertHistory(this, task.Execution, atxe); task.Execution.AwaitingDecision = true; TaskLists.DeleteOnSubmit(task); break; case "heartbeat": if (task != null) { if (taskResponse.Progress.HasValue) { task.Progress = taskResponse.Progress.Value; } if (taskResponse.ProgressMessage != null) { task.ProgressMessage = taskResponse.ProgressMessage; } if (task.HeartbeatTimeout.HasValue) { task.HeartbeatAlarm = DateTime.UtcNow.AddSeconds(task.HeartbeatTimeout.Value); } // Send a progress notification to the updater var data = new JObject( new JProperty("type", "ActivityTaskHeartbeat"), new JProperty("progress", task.Progress ?? -1), new JProperty("message", task.ProgressMessage ?? ""), new JProperty("progressData", JToken.Parse(task.ProgressData))); var qt = CreateUpdaterNotification(task.Execution, data); TaskLists.InsertOnSubmit(qt); SubmitChanges(); } else { log.Info($"Received heartbeat for deleted task with token: {taskToken}, cancelling"); } // If the task has been cancelled or deleted then send a cancellation request response = new JObject { new JProperty("cancellationRequested", task == null || task.Cancelling) }.ToString(Formatting.None); break; case "rescheduled": // Rescheduled tasks are left in the tasklist with an updated scheduling time // effectively pushing them to the back of the queue if (task != null) { task.ScheduledAt = DateTime.UtcNow; task.WorkerId = null; } break; default: log.ErrorFormat("Invalid activity task response status: {0}", taskResponse.Status); break; } // Nothing to update if the task has been deleted if (task == null) { return(response); } try { SubmitChanges(); break; } catch (ChangeConflictException) { Refresh(RefreshMode.KeepCurrentValues, task.Execution); } } if (i == RETRIES) { log.Error($"Failed to process {taskResponse.Status} response from task {taskToken}"); return(null); } return(response); } catch (Exception ex) { log.Error("Failed to process activity task response", ex); return(null); } }