NodeTask Visit(Node node, NodeTasks tasks, int tick, CancellationToken token) { // If there's no task associated with this node yet, create one by waiting on all tasks of its inputs and throwing their results into Process // Also wait on the node's previous task if existing to avoid race conditions NodeTask result; if (!tasks.TryGetValue(node, out result)) { var dependencies = node.Inputs.Where(i => i.Source != null).Select(i => new { Output = i.Source, Task = Visit(i.Source.Node, tasks, tick, token) }).ToArray(); Func <NodeTask> getResult = () => ContinueWhenAll( dependencies.Select(dep => dep.Task).ToArray(), _ => node.Process(dependencies.Select(dep => dep.Task.Result[dep.Output.Index]).ToArray(), tick), token ); NodeTask previous; if (previousTasks.TryGetValue(node, out previous)) { result = previous.ContinueWith(_ => getResult(), token, TaskContinuationOptions.AttachedToParent, TaskScheduler.Current).Unwrap(); } else { result = getResult(); } tasks.Add(node, result); } return(result); }
Task <FrameDic> RenderTickCore(IEnumerable <Node> startNodes, int tick, CancellationToken token) { // Start task for each start task, wait on their completion, zip together their outputs and the corresponding computed frames. NodeTasks allTasks = new NodeTasks(); var startTasks = startNodes.Select(start => new { Outputs = start.Outputs, Task = Visit(start, allTasks, tick, token) }).ToArray(); previousTasks = allTasks; return(ContinueWhenAll( startTasks.Select(t => t.Task).ToArray(), _ => (FrameDic)startTasks.SelectMany(t => t.Outputs.Zip(t.Task.Result, Tuple.Create)).ToDictionary(tup => tup.Item1, tup => tup.Item2), token )); }
private async Async.Task <Error?> OnWorkerEventRunning(Guid machineId, WorkerRunningEvent running) { var(task, node) = await( _context.TaskOperations.GetByTaskId(running.TaskId), _context.NodeOperations.GetByMachineId(machineId)); if (task is null) { return(new Error( Code: ErrorCode.INVALID_REQUEST, Errors: new string[] { $"unable to find task: {running.TaskId}" })); } if (node is null) { return(new Error( Code: ErrorCode.INVALID_REQUEST, Errors: new string[] { $"unable to find node: {machineId}" })); } if (!node.State.ReadyForReset()) { await _context.NodeOperations.SetState(node, NodeState.Busy); } var nodeTask = new NodeTasks( MachineId: machineId, TaskId: running.TaskId, State: NodeTaskState.Running); await _context.NodeTasksOperations.Replace(nodeTask); if (task.State.ShuttingDown()) { _log.Info($"ignoring task start from node. machine_id:{machineId} job_id:{task.JobId} task_id:{task.TaskId} (state: {task.State})"); return(null); } _log.Info($"task started on node. machine_id:{machineId} job_id:{task.JobId} task_id:{task.TaskId}"); await _context.TaskOperations.SetState(task, TaskState.Running); var taskEvent = new TaskEvent( TaskId: task.TaskId, MachineId: machineId, EventData: new WorkerEvent(Running: running)); await _context.TaskEventOperations.Replace(taskEvent); return(null); }
public bool NodeTasks(NodeTasks e) { return(Test(e)); }
private async Async.Task <Error?> OnStateUpdate(Guid machineId, NodeStateUpdate ev) { var node = await _context.NodeOperations.GetByMachineId(machineId); if (node is null) { _log.Warning($"unable to process state update event. machine_id:{machineId} state event:{ev}"); return(null); } if (ev.State == NodeState.Free) { if (node.ReimageRequested || node.DeleteRequested) { _log.Info($"stopping free node with reset flags: {machineId}"); await _context.NodeOperations.Stop(node); return(null); } if (await _context.NodeOperations.CouldShrinkScaleset(node)) { _log.Info($"stopping free node to resize scaleset: {machineId}"); await _context.NodeOperations.SetHalt(node); return(null); } } if (ev.State == NodeState.Init) { if (node.DeleteRequested) { _log.Info($"stopping node (init and delete_requested): {machineId}"); await _context.NodeOperations.Stop(node); return(null); } // Don’t check reimage_requested, as nodes only send 'init' state once. If // they send 'init' with reimage_requested, it's because the node was reimaged // successfully. node = node with { ReimageRequested = false, InitializedAt = DateTimeOffset.UtcNow }; await _context.NodeOperations.SetState(node, ev.State); return(null); } _log.Info($"node state update: {machineId} from {node.State} to {ev.State}"); await _context.NodeOperations.SetState(node, ev.State); if (ev.State == NodeState.Free) { _log.Info($"node now available for work: {machineId}"); } else if (ev.State == NodeState.SettingUp) { if (ev.Data is NodeSettingUpEventData settingUpData) { if (!settingUpData.Tasks.Any()) { return(new Error(ErrorCode.INVALID_REQUEST, Errors: new string[] { $"setup without tasks. machine_id: {machineId}", })); } foreach (var taskId in settingUpData.Tasks) { var task = await _context.TaskOperations.GetByTaskId(taskId); if (task is null) { return(new Error( ErrorCode.INVALID_REQUEST, Errors: new string[] { $"unable to find task: {taskId}" })); } _log.Info($"node starting task. machine_id: {machineId} job_id: {task.JobId} task_id: {task.TaskId}"); // The task state may be `running` if it has `vm_count` > 1, and // another node is concurrently executing the task. If so, leave // the state as-is, to represent the max progress made. // // Other states we would want to preserve are excluded by the // outermost conditional check. if (task.State != TaskState.Running && task.State != TaskState.SettingUp) { await _context.TaskOperations.SetState(task, TaskState.SettingUp); } var nodeTask = new NodeTasks( MachineId: machineId, TaskId: task.TaskId, State: NodeTaskState.SettingUp); await _context.NodeTasksOperations.Replace(nodeTask); } } } else if (ev.State == NodeState.Done) { Error?error = null; if (ev.Data is NodeDoneEventData doneData) { if (doneData.Error is not null) { var errorText = EntityConverter.ToJsonString(doneData); error = new Error(ErrorCode.TASK_FAILED, Errors: new string[] { errorText }); _log.Error($"node 'done' with error: machine_id:{machineId}, data:{errorText}"); } } // if tasks are running on the node when it reports as Done // those are stopped early await _context.NodeOperations.MarkTasksStoppedEarly(node, error); await _context.NodeOperations.ToReimage(node, done : true); } return(null); }