/// <summary>
/// When a task fails, follow the error outflow if one exists else fail the workflow
/// </summary>
/// <param name="db"></param>
/// <param name="workflow"></param>
/// <param name="execution"></param>
/// <param name="history"></param>
/// <param name="decisions"></param>
private static void ProcessActivityTaskFailedEvent(Database db, WorkflowObj workflow, Execution execution, History history, List <History> decisions)
{
// There should(!) be no contention for the data modified in this process
var evt = ActivityTaskFailedEvent.Create(history);
var se = ActivityTaskScheduledEvent.Create(db.GetHistoryEvent(execution.ExecutionId, evt.SchedulingEventId));
var failedTask = workflow.Tasks.Single(t => t.TaskId == se.TaskId);
var nextTaskId = failedTask.FailOutflow?.Target;
if (!string.IsNullOrEmpty(nextTaskId))
{
var nextTask = workflow.Tasks.Single(t => t.TaskId == nextTaskId);
CreateTaskScheduledEvent(db, execution, nextTask, decisions);
}
decisions.Add(new WorkflowExecutionFailedEvent
{
Reason = $"Task {se.TaskId} failed with no recovery action defined"
});
AttemptCleanup(db, execution, workflow, decisions);
}
/// <summary>
/// Create a task scheduled event and add it to the decision list
/// </summary>
/// <param name="db"></param>
/// <param name="execution"></param>
/// <param name="task"></param>
/// <param name="decisions"></param>
private static void CreateTaskScheduledEvent(Database db, Execution execution, TaskObj task, ICollection <History> decisions)
{
var act = db.GetActivity(task.ActivityName, task.ActivityVersion);
var input = GetTaskInput(execution, task);
var activity = JsonConvert.DeserializeObject <ActivityObj>(act.Json);
var atse = new ActivityTaskScheduledEvent
{
ActivityId = Guid.NewGuid().ToString(),
ActivityName = task.ActivityName,
ActivityVersion = task.ActivityVersion,
TaskList = task.TaskList ?? activity.DefaultTaskList, // TODO: Should this be overridable in the task definition? Probably!
TaskId = task.TaskId,
AsyncSignal = task.AsyncSignal,
TaskPriority = task.TaskPriority ?? activity.DefaultPriority ?? 0,
HeartbeatTimeout = (int?)(task.HeartbeatTimeout ?? activity.DefaultTaskHeartbeatTimeout),
ScheduleToCloseTimeout = (int?)(task.ScheduleToCloseTimeout ?? activity.DefaultTaskScheduleToCloseTimeout),
ScheduleToStartTimeout = (int?)(task.ScheduleToStartTimeout ?? activity.DefaultTaskScheduleToStartTimeout),
StartToCloseTimeout = (int?)task.StartToCloseTimeout,
Input = input
};
decisions.Add(atse);
}
/// <summary>
/// Respond to a request for work by checking for a suitable waiting task
/// If one is found, set up the timeout alarms, add the task started event and return the task to the worker
/// </summary>
/// <param name="workerId">The worker polling</param>
/// <param name="taskListName">The tasklist to poll</param>
/// <returns>The task, if one is available</returns>
public string SelectAndStartTask(string workerId, string taskListName)
{
var task =
(from t in TaskLists
where t.ListName == taskListName && t.WorkerId == null
orderby t.Priority, t.ScheduledAt
select t).FirstOrDefault();
// Nothing to do
if (task == null)
{
return(null);
}
// Check for a notification
if (task.TaskScheduledEventId == 0)
{
var notifyTask = new ActivityTask
{
ActivityId = null,
ActivityName = "$notify",
ActivityVersion = "1.0.0.0",
ExecutionId = task.ExecutionId,
JobId = task.JobId,
AsyncSignal = null,
Input = JObject.Parse(task.NotificationData),
TaskToken = task.TaskToken,
StartedEventId = 0
};
TaskLists.DeleteOnSubmit(task);
SubmitChanges();
return(JsonConvert.SerializeObject(notifyTask));
}
task.WorkerId = workerId;
task.StartedAt = DateTime.UtcNow;
if (task.HeartbeatTimeout.HasValue)
{
task.HeartbeatAlarm = DateTime.UtcNow.AddSeconds(task.HeartbeatTimeout.Value);
}
var scheduleToCloseAlarm = DateTime.MinValue;
if (task.TaskSheduleToCloseTimeout.HasValue)
{
scheduleToCloseAlarm = task.ScheduledAt.AddSeconds(task.TaskSheduleToCloseTimeout.Value);
}
var startToCloseAlarm = DateTime.MinValue;
if (task.TaskStartToCloseTimeout.HasValue)
{
startToCloseAlarm = task.ScheduledAt.AddSeconds(task.TaskStartToCloseTimeout.Value);
}
// Select the earliest alarm time
var alarm = scheduleToCloseAlarm < startToCloseAlarm ? scheduleToCloseAlarm : startToCloseAlarm;
if (alarm > DateTime.MinValue)
{
task.TaskAlarm = alarm;
}
try
{
SubmitChanges();
}
catch (ChangeConflictException ex)
{
// The likely suspects are:
// 1) Task timeout - OK, we were just too late
// 2) Cancellation - OK we were just late enough
// 3) Task taken by other worker - No problem
// In all cases there is no error so we just bail
log.Info("ChangeConflictException in SelectAndStartTask", ex);
return(null);
}
var atse = new ActivityTaskStartedEvent
{
WorkerId = workerId,
ScheduledEventId = task.TaskScheduledEventId,
};
InsertHistory(this, task.Execution, atse);
SubmitChanges();
// Get the scheduling event
var evt = ActivityTaskScheduledEvent.Create(Histories.Single(h => (h.ExecutionId == task.ExecutionId) && (h.Id == task.TaskScheduledEventId)));
var at = new ActivityTask
{
ActivityId = evt.ActivityId,
ActivityName = evt.ActivityName,
ActivityVersion = evt.ActivityVersion,
ExecutionId = evt.ExecutionId,
JobId = task.JobId,
AsyncSignal = evt.AsyncSignal,
Input = evt.Input,
TaskToken = task.TaskToken,
StartedEventId = atse.Id
};
return(JsonConvert.SerializeObject(at));
}
/// <summary>
/// Store decisions and create a Tasklist item for each ActivityTaskScheduledEvents
/// </summary>
/// <param name="db"></param>
/// <param name="execution"></param>
/// <param name="decisions"></param>
/// <param name="historySeen"></param>
/// <returns></returns>
private static bool StoreDecisions(Database db, Execution execution, IEnumerable <History> decisions, int historySeen)
{
foreach (var decision in decisions)
{
// Grab the history
var historyId = Database.InsertHistory(execution, decision);
TaskList qt;
if (ActivityTaskScheduledEvent.CanCreate(decision))
{
var atse = ActivityTaskScheduledEvent.Create(decision);
const int YEAR = 3600 * 12 * 365;
qt = new TaskList
{
ExecutionId = execution.ExecutionId,
JobId = execution.JobId,
ListName = atse.TaskList,
TaskToken = Guid.NewGuid(),
TaskScheduledEventId = historyId,
Priority = atse.TaskPriority,
HeartbeatTimeout = atse.HeartbeatTimeout,
TaskAlarm = DateTime.UtcNow.AddSeconds(Math.Min(atse.ScheduleToCloseTimeout ?? YEAR, atse.ScheduleToStartTimeout ?? YEAR)),
ScheduledAt = DateTime.UtcNow,
TaskSheduleToCloseTimeout = atse.ScheduleToCloseTimeout,
TaskStartToCloseTimeout = atse.StartToCloseTimeout,
ProgressData = (atse.Input.SelectToken("progressData") ?? JValue.CreateNull()).ToString(Formatting.None)
};
db.TaskLists.InsertOnSubmit(qt);
}
// Notify updater
else if (WorkflowExecutionFailedEvent.CanCreate(decision))
{
var wefe = WorkflowExecutionFailedEvent.Create(decision);
var data = new JObject(
new JProperty("type", wefe.EventType),
new JProperty("reason", wefe.Reason));
qt = Database.CreateUpdaterNotification(execution, data);
db.TaskLists.InsertOnSubmit(qt);
}
else if (WorkflowCleanupStartedEvent.CanCreate(decision))
{
execution.ExecutionState.State = ExState.Cleanup;
}
}
try
{
if (historySeen != 0)
{
execution.HistorySeen = historySeen;
}
execution.DeciderToken = null;
execution.LastSeen = DateTime.UtcNow;
db.SubmitChanges();
}
catch (ChangeConflictException)
{
db.Refresh(RefreshMode.KeepCurrentValues, execution);
db.SubmitChanges();
}
return(true);
}
/// <summary>
/// Process task completion by scheduling the next task or completing the workflow
/// </summary>
/// <param name="db"></param>
/// <param name="workflow"></param>
/// <param name="execution"></param>
/// <param name="history"></param>
/// <param name="decisions"></param>
private static void ProcessActivityTaskCompletedEvent(Database db, WorkflowObj workflow, Execution execution, History history, List <History> decisions)
{
// There should(!) be no contention for the data modified in this process
var evt = ActivityTaskCompletedEvent.Create(history);
var se = ActivityTaskScheduledEvent.Create(db.GetHistoryEvent(execution.ExecutionId, evt.SchedulingEventId));
var completedTask = workflow.Tasks.Single(t => t.TaskId == se.TaskId);
// Default task outflow
var outflow = "Out";
// Update variables
if (evt.Result != null)
{
// Update the variables if there are any normal results (not prefixed with "$")
if (evt.Result.Properties().Any(p => !p.Name.StartsWith("$")))
{
var variables = db.Variables.Where(v => v.Execution == execution).ToArray();
foreach (var o in completedTask.Outputs.Where(o => o.Value.Var != null))
{
// If the activity has not returned a value for an output then we don't update the mapped variable
// In general it is probably best if activities return values for all outputs to avoid confusion
JToken value;
if (evt.Result.TryGetValue(o.Key, out value))
{
variables.Single(v => v.Name == o.Value.Var).Json = value.ToString(Formatting.None);
}
}
}
// Get the correct outflow
JToken outflowToken;
if (evt.Result.TryGetValue("$outflow", out outflowToken))
{
if (outflowToken.Type == JTokenType.String)
{
outflow = (string)outflowToken;
}
else
{
throw new ApplicationException("Task outflow identifier must be a string");
}
}
}
var nextTaskId = completedTask.Outflows.Single(o => o.Name == outflow).Target;
var nextTask = workflow.Tasks.Single(t => t.TaskId == nextTaskId);
// A task with no outflows is an end
if (nextTask.Outflows.Length == 0)
{
Console.WriteLine($"Execution state = {execution.ExecutionState.State}");
if (ExState.Create(execution.ExecutionState.State) != ExState.Cleanup)
{
CreateWorkflowCompletedEvent(execution, decisions);
}
AttemptCleanup(db, execution, workflow, decisions);
}
else
{
CreateTaskScheduledEvent(db, execution, nextTask, decisions);
}
}
