NodeTask Visit(Node node, NodeTasks tasks, int tick, CancellationToken token)
{
// If there's no task associated with this node yet, create one by waiting on all tasks of its inputs and throwing their results into Process
// Also wait on the node's previous task if existing to avoid race conditions
NodeTask result;
if (!tasks.TryGetValue(node, out result))
{
var dependencies = node.Inputs.Where(i => i.Source != null).Select(i => new { Output = i.Source, Task = Visit(i.Source.Node, tasks, tick, token) }).ToArray();
Func <NodeTask> getResult = () => ContinueWhenAll(
dependencies.Select(dep => dep.Task).ToArray(),
_ => node.Process(dependencies.Select(dep => dep.Task.Result[dep.Output.Index]).ToArray(), tick),
token
);
NodeTask previous;
if (previousTasks.TryGetValue(node, out previous))
{
result = previous.ContinueWith(_ => getResult(), token, TaskContinuationOptions.AttachedToParent, TaskScheduler.Current).Unwrap();
}
else
{
result = getResult();
}
tasks.Add(node, result);
}
return(result);
}
Task <FrameDic> RenderTickCore(IEnumerable <Node> startNodes, int tick, CancellationToken token)
{
// Start task for each start task, wait on their completion, zip together their outputs and the corresponding computed frames.
NodeTasks allTasks = new NodeTasks();
var startTasks = startNodes.Select(start => new { Outputs = start.Outputs, Task = Visit(start, allTasks, tick, token) }).ToArray();
previousTasks = allTasks;
return(ContinueWhenAll(
startTasks.Select(t => t.Task).ToArray(),
_ => (FrameDic)startTasks.SelectMany(t => t.Outputs.Zip(t.Task.Result, Tuple.Create)).ToDictionary(tup => tup.Item1, tup => tup.Item2),
token
));
}
private async Async.Task <Error?> OnWorkerEventRunning(Guid machineId, WorkerRunningEvent running)
{
var(task, node) = await(
_context.TaskOperations.GetByTaskId(running.TaskId),
_context.NodeOperations.GetByMachineId(machineId));
if (task is null)
{
return(new Error(
Code: ErrorCode.INVALID_REQUEST,
Errors: new string[] { $"unable to find task: {running.TaskId}" }));
}
if (node is null)
{
return(new Error(
Code: ErrorCode.INVALID_REQUEST,
Errors: new string[] { $"unable to find node: {machineId}" }));
}
if (!node.State.ReadyForReset())
{
await _context.NodeOperations.SetState(node, NodeState.Busy);
}
var nodeTask = new NodeTasks(
MachineId: machineId,
TaskId: running.TaskId,
State: NodeTaskState.Running);
await _context.NodeTasksOperations.Replace(nodeTask);
if (task.State.ShuttingDown())
{
_log.Info($"ignoring task start from node. machine_id:{machineId} job_id:{task.JobId} task_id:{task.TaskId} (state: {task.State})");
return(null);
}
_log.Info($"task started on node. machine_id:{machineId} job_id:{task.JobId} task_id:{task.TaskId}");
await _context.TaskOperations.SetState(task, TaskState.Running);
var taskEvent = new TaskEvent(
TaskId: task.TaskId,
MachineId: machineId,
EventData: new WorkerEvent(Running: running));
await _context.TaskEventOperations.Replace(taskEvent);
return(null);
}
public bool NodeTasks(NodeTasks e)
{
return(Test(e));
}
private async Async.Task <Error?> OnStateUpdate(Guid machineId, NodeStateUpdate ev)
{
var node = await _context.NodeOperations.GetByMachineId(machineId);
if (node is null)
{
_log.Warning($"unable to process state update event. machine_id:{machineId} state event:{ev}");
return(null);
}
if (ev.State == NodeState.Free)
{
if (node.ReimageRequested || node.DeleteRequested)
{
_log.Info($"stopping free node with reset flags: {machineId}");
await _context.NodeOperations.Stop(node);
return(null);
}
if (await _context.NodeOperations.CouldShrinkScaleset(node))
{
_log.Info($"stopping free node to resize scaleset: {machineId}");
await _context.NodeOperations.SetHalt(node);
return(null);
}
}
if (ev.State == NodeState.Init)
{
if (node.DeleteRequested)
{
_log.Info($"stopping node (init and delete_requested): {machineId}");
await _context.NodeOperations.Stop(node);
return(null);
}
// Don’t check reimage_requested, as nodes only send 'init' state once. If
// they send 'init' with reimage_requested, it's because the node was reimaged
// successfully.
node = node with {
ReimageRequested = false, InitializedAt = DateTimeOffset.UtcNow
};
await _context.NodeOperations.SetState(node, ev.State);
return(null);
}
_log.Info($"node state update: {machineId} from {node.State} to {ev.State}");
await _context.NodeOperations.SetState(node, ev.State);
if (ev.State == NodeState.Free)
{
_log.Info($"node now available for work: {machineId}");
}
else if (ev.State == NodeState.SettingUp)
{
if (ev.Data is NodeSettingUpEventData settingUpData)
{
if (!settingUpData.Tasks.Any())
{
return(new Error(ErrorCode.INVALID_REQUEST, Errors: new string[] {
$"setup without tasks. machine_id: {machineId}",
}));
}
foreach (var taskId in settingUpData.Tasks)
{
var task = await _context.TaskOperations.GetByTaskId(taskId);
if (task is null)
{
return(new Error(
ErrorCode.INVALID_REQUEST,
Errors: new string[] { $"unable to find task: {taskId}" }));
}
_log.Info($"node starting task. machine_id: {machineId} job_id: {task.JobId} task_id: {task.TaskId}");
// The task state may be `running` if it has `vm_count` > 1, and
// another node is concurrently executing the task. If so, leave
// the state as-is, to represent the max progress made.
//
// Other states we would want to preserve are excluded by the
// outermost conditional check.
if (task.State != TaskState.Running && task.State != TaskState.SettingUp)
{
await _context.TaskOperations.SetState(task, TaskState.SettingUp);
}
var nodeTask = new NodeTasks(
MachineId: machineId,
TaskId: task.TaskId,
State: NodeTaskState.SettingUp);
await _context.NodeTasksOperations.Replace(nodeTask);
}
}
}
else if (ev.State == NodeState.Done)
{
Error?error = null;
if (ev.Data is NodeDoneEventData doneData)
{
if (doneData.Error is not null)
{
var errorText = EntityConverter.ToJsonString(doneData);
error = new Error(ErrorCode.TASK_FAILED, Errors: new string[] { errorText });
_log.Error($"node 'done' with error: machine_id:{machineId}, data:{errorText}");
}
}
// if tasks are running on the node when it reports as Done
// those are stopped early
await _context.NodeOperations.MarkTasksStoppedEarly(node, error);
await _context.NodeOperations.ToReimage(node, done : true);
}
return(null);
}