public void CanResetOwnTypedTasks()
        {
            var db = _mux.GetDatabase();

            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th  = new BlockingTaskHandler("1");
            var th2 = new BlockingTaskHandler("2");

            var complete = new ManualResetEvent(false);

            var events = new List <string>();

            var wh = new WorkflowHandler();

            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var t1 = new TaskName("TestNode1");
            var t2 = new TaskName("TestNode2");
            var t3 = new TaskName("TestNode3");
            var t4 = new TaskName("TestNode4");
            var t5 = new TaskName("TestNode5");
            var t6 = new TaskName("TestNode6");

            var type1 = new TaskType("testTaskType");

            var workflow = new Workflow(new WorkflowName("TestWorkflow"));

            workflow.AddTask(t1, new Payload("Node1"), type1, SimplePriority, EmptyTaskList, new[] { t2 });
            workflow.AddTask(t2, new Payload("Node2"), type1, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
            workflow.AddTask(t3, new Payload("Node3"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t4, new Payload("Node4"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t5, new Payload("Node5"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t6, new Payload("Node6"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), new[] { type1 }, new Lua()))
            {
                wm.PushWorkflow(workflow);

                // we've picked up the first task, and don't want to pick up any more
                th.Gate.WaitOne();

                Assert.AreEqual(0, db.SortedSetLength("submitted:testTaskType"));
            }

            // create a new wm to simulate a restart of a component
            using (var wm2 = new WorkflowManagement(_mux, th2, wh, new WorkflowManagementId("test"), new[] { type1 }, new Lua()))
            {
                Assert.AreEqual(1, db.SortedSetLength("submitted:testTaskType"));

                th.Abort.Set();
                th2.LetRun.Set();

                var result = complete.WaitOne();
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
Ejemplo n.º 2
0
        internal Task(WorkflowName workflowId, TaskName name, IEnumerable <TaskName> parents, IEnumerable <TaskName> children, Payload payload, TaskType type, TaskPriority priority)
        {
            Workflow = workflowId;
            Name     = name;
            Payload  = payload;
            Type     = type;
            Priority = priority;

            _parents = parents.ToArray();

            _children = children.ToArray();
        }
Ejemplo n.º 3
0
        internal Task(WorkflowName workflowId, TaskName name, IEnumerable<TaskName> parents, IEnumerable<TaskName> children, Payload payload, TaskType type, TaskPriority priority)
        {
            Workflow = workflowId;
            Name = name;
            Payload = payload;
            Type = type;
            Priority = priority;

            _parents = parents.ToArray();

            _children = children.ToArray();
        }
        public void CanSubmitAndRunAWorkflowWithTypedTasks()
        {
            var db = _mux.GetDatabase();

            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new TestTaskHandler();

            var complete = new ManualResetEvent(false);

            var events = new List <string>();

            var wh = new WorkflowHandler();

            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var type1 = new TaskType("testTaskType");
            var type2 = new TaskType("testTaskType2");

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), new[] { type1 }, new Lua()))
                using (var wm2 = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test2"), new[] { type2 }, new Lua()))
                {
                    var t1 = new TaskName("TestNode1");
                    var t2 = new TaskName("TestNode2");
                    var t3 = new TaskName("TestNode3");
                    var t4 = new TaskName("TestNode4");
                    var t5 = new TaskName("TestNode5");
                    var t6 = new TaskName("TestNode6");

                    var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                    workflow.AddTask(t1, new Payload("Node1"), type1, SimplePriority, EmptyTaskList, new[] { t2 });
                    workflow.AddTask(t2, new Payload("Node2"), type1, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
                    workflow.AddTask(t3, new Payload("Node3"), type2, SimplePriority, new[] { t2 }, EmptyTaskList);
                    workflow.AddTask(t4, new Payload("Node4"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
                    workflow.AddTask(t5, new Payload("Node5"), type2, SimplePriority, new[] { t2 }, EmptyTaskList);
                    workflow.AddTask(t6, new Payload("Node6"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);

                    wm.PushWorkflow(workflow);

                    var workflowWasCompleted = complete.WaitOne(2000);

                    Assert.IsTrue(workflowWasCompleted);

                    Console.WriteLine("WM events"); foreach (var ev in events)
                    {
                        Console.WriteLine("Event: " + ev);
                    }

                    Assert.AreEqual(6, th.TaskRunCount);
                }
        }
        public void CanReleaseAWorkflow()
        {
            var db = _mux.GetDatabase();

            db.ScriptEvaluate("print(\"CanPauseAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler("1");

            var complete = new ManualResetEvent(false);

            var events = new List <string>();

            var wh = new WorkflowHandler();

            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var t1 = new TaskName("TestNode1");
            var t2 = new TaskName("TestNode2");
            var t3 = new TaskName("TestNode3");
            var t4 = new TaskName("TestNode4");
            var t5 = new TaskName("TestNode5");
            var t6 = new TaskName("TestNode6");

            var type1 = new TaskType("testTaskType");

            var workflow = new Workflow(new WorkflowName("TestWorkflow"));

            workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
            workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, EmptyTaskList, new[] { t3, t4, t5, t6 });
            workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var workflowId = wm.PushWorkflow(workflow);

                th.Gate.WaitOne();

                wm.PauseWorkflow(workflowId);

                wm.ReleaseWorkflow(workflowId);

                Assert.AreEqual(1, db.ListLength("submitted"));
                Assert.AreEqual(1, db.SetLength("running"));
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
Ejemplo n.º 6
0
        internal void Run(string[] args)
        {
            // Should you want to decompose to running single tasks you can just create a workflow containing nodes with no parents or
            // children. The system works out where to start by identifying nodes with no parents, and assuming they're immediately
            // runnable. The hosting workflow then gives you a handle by which you can treat the individual tasks as a related set.
            var t1 = new TaskName("TestNode1");
            var t2 = new TaskName("TestNode2");
            var t3 = new TaskName("TestNode3");
            var t4 = new TaskName("TestNode4");
            var t5 = new TaskName("TestNode5");
            var t6 = new TaskName("TestNode6");

            var type1 = new TaskType("testTaskType");

            var workflow = new Common.Workflow(new WorkflowName("TestWorkflow"));
            workflow.AddTask(t1, new Payload("Node1"), type1, SimplePriority, EmptyTaskList, new[] { t2 });
            workflow.AddTask(t2, new Payload("Node2"), type1, SimplePriority, EmptyTaskList, new[] { t3, t4, t5, t6 });
            workflow.AddTask(t3, new Payload("Node3"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t4, new Payload("Node4"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t5, new Payload("Node5"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t6, new Payload("Node6"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);

            // A client will start acting on tasks immediately. May not want that to
            // be behaviour -- so might want to have an independent submit-only client. TaskHandler only needed for
            // submission. TaskHandler needs to be stateless, taking configuration for a specific task on a run call? No, that's
            // true per-task, but agiven ITaskHandler instance might be configured once wth e.g. compute resource connection details
            var th = new TaskHandler();
            var wh = new WorkflowHandler();
            var complete = new ManualResetEvent(false);
            var failed = new ManualResetEvent(false);
            wh.WorkflowComplete += (s, w) => { Console.WriteLine("workflow complete: " + w); complete.Set(); };
            wh.WorkflowFailed += (s, w) => { Console.WriteLine("workflow failed: " + w); failed.Set(); };

            EventHandler<Exception> eh = (s, e) => { };

            var wm = new WorkflowManagement(ConnectionMultiplexer.Connect("localhost"), th, wh, new WorkflowManagementId("sampleApp"), eh);

            wm.ClearBacklog();

            // Pushing a workflow causes it to be executed
            var id = wm.PushWorkflow(workflow);

            Console.WriteLine("Workflow pushed");

            WaitHandle.WaitAny(new[] { failed, complete });

            wm.CleanUp(id.ToString());

            Console.ReadLine();
        }
        public void HandlesExceptionsFromBadLuaInAFailingWorkflow()
        {
            var cases = new[] { TestCase.FailTask, TestCase.PopFailedWorkflow };

            foreach (var testCase in cases)
            {
                var lua = new BadLua(testCase);

                var message = "HandlesExceptionsFromBadLuaInAFailingWorkflow:" + testCase;

                var db = _mux.GetDatabase();
                db.ScriptEvaluate("print(\"" + message + "\")");
                db.ScriptEvaluate("redis.call(\"flushdb\")");

                var th = new ErroringTestWithLongRunnerTaskHandler("Node4", "Node3");

                var complete  = new ManualResetEvent(false);
                var failed    = new ManualResetEvent(false);
                var exception = new ManualResetEvent(false);

                var events = new List <string>();

                var wh = new WorkflowHandler();
                wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
                wh.WorkflowFailed   += (s, w) => { events.Add("failed"); failed.Set(); };

                EventHandler <Exception> eh = (s, e) => { events.Add(e.Message); exception.Set(); };

                using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, lua, eh, Behaviours.Processor | Behaviours.Submitter))
                {
                    var t1 = new TaskName("TestNode1");
                    var t2 = new TaskName("TestNode2");
                    var t3 = new TaskName("TestNode3");
                    var t4 = new TaskName("TestNode4");

                    var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                    workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new [] { t2 });
                    workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new [] { t1 }, new [] { t3, t4 });
                    workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
                    workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);

                    wm.PushWorkflow(workflow);

                    var waitResult = WaitHandle.WaitAny(new[] { exception, failed, complete }, 2000);

                    Assert.AreEqual(0, waitResult);
                }
            }
        }
Ejemplo n.º 8
0
        public void AddTask(TaskName name, Payload payload, TaskType type, TaskPriority priority, IEnumerable<TaskName> parents, IEnumerable<TaskName> children)
        {
            if (!_defined.Add(name))
            {
                throw new WorkflowException(string.Format("Task '{0}' has already been declared.", name));
            }

            foreach (var parent in parents) _parents.Add(parent);

            foreach (var child in children) _children.Add(child);

            var task = new Task(Name, name, parents, children, payload, type, priority);

            _tasks.Add(task);
        }
        public void CanSubmitAWorkflow()
        {
            var db = _mux.GetDatabase();

            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler();

            th.LetRun.Set();

            var complete = new ManualResetEvent(false);

            var events = new List <string>();

            var wh = new WorkflowHandler();

            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");
                var t5 = new TaskName("TestNode5");
                var t6 = new TaskName("TestNode6");

                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
                workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

                wm.PushWorkflow(workflow);

                var result = WaitHandle.WaitAny(new[] { complete }, 2000);

                Assert.AreEqual(0, result);
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
Ejemplo n.º 10
0
        public void AddTask(TaskName name, Payload payload, TaskType type, TaskPriority priority, IEnumerable <TaskName> parents, IEnumerable <TaskName> children)
        {
            if (!_defined.Add(name))
            {
                throw new WorkflowException(string.Format("Task '{0}' has already been declared.", name));
            }

            foreach (var parent in parents)
            {
                _parents.Add(parent);
            }

            foreach (var child in children)
            {
                _children.Add(child);
            }

            var task = new Task(Name, name, parents, children, payload, type, priority);

            _tasks.Add(task);
        }
Ejemplo n.º 11
0
        public void IfATaskIsMarkedFailed_WorfklowFailsAndNoMoreTasksSubmitted()
        {
            var db = _mux.GetDatabase();

            db.ScriptEvaluate("print(\"IfATaskIsMarkedFailed_WorfklowFailsAndNoMoreTasksSubmitted\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new ErroringTestTaskHandler(2);

            var complete = new ManualResetEvent(false);
            var failed   = new ManualResetEvent(false);

            var wh = new WorkflowHandler();

            wh.WorkflowComplete += (s, w) => { complete.Set(); };
            wh.WorkflowFailed   += (s, w) => { failed.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");

                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4 });
                workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

                wm.PushWorkflow(workflow);

                var result = WaitHandle.WaitAny(new[] { failed, complete }, 2000);

                Assert.AreEqual(0, result);
                Assert.AreEqual(2, th.TaskRunCount); // only 2 get sent through the task handler; second fails, so no children are executed
            }
        }
Ejemplo n.º 12
0
        public void CanPauseAWorkflow()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanPauseAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler("1");

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var t1 = new TaskName("TestNode1");
            var t2 = new TaskName("TestNode2");
            var t3 = new TaskName("TestNode3");
            var t4 = new TaskName("TestNode4");
            var t5 = new TaskName("TestNode5");
            var t6 = new TaskName("TestNode6");

            var type1 = new TaskType("testTaskType");

            var workflow = new Workflow(new WorkflowName("TestWorkflow"));
            workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
            workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, EmptyTaskList, new[] { t3, t4, t5, t6 });
            workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var workflowId = wm.PushWorkflow(workflow);

                var waitResult = th.Gate.WaitOne(/*2000*/);

                Assert.IsTrue(waitResult);
                Assert.AreEqual(1, db.SortedSetLength("submitted"));
                Assert.AreEqual(1, db.SetLength("running"));

                wm.PauseWorkflow(workflowId);

                var expected = new List<string> { "1", "2" };
                var state = "paused";

                CheckSetContent(db, expected, state);

                // The running task will run bang into the new state
                th.LetRun.Set();

                Thread.Sleep(500); // gah, honestly. I won't get a signal because no new tasks should be submitted

                expected = new List<string> { "2" };
                state = "paused";

                CheckSetContent(db, expected, state);

                expected = new List<string> { "2" };
                state = "running";

                CheckSetContent(db, expected, state, false);

                // If this is checked before the task has completed then there'll be two paused tasks
                // We've got no hook into the completion event as yet
                Assert.AreEqual(1, db.SetLength("paused")); 
                Assert.AreEqual(1, db.SetLength("paused:1"));
                Assert.AreEqual(0, db.SortedSetLength("submitted"));
                Assert.AreEqual(0, db.SetLength("running"));
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
Ejemplo n.º 13
0
        public void CanReleaseAWorkflowWithTypedTasks()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanPauseAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler("1");

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var t1 = new TaskName("TestNode1");
            var t2 = new TaskName("TestNode2");
            var t3 = new TaskName("TestNode3");
            var t4 = new TaskName("TestNode4");
            var t5 = new TaskName("TestNode5");
            var t6 = new TaskName("TestNode6");

            var type1 = new TaskType("testTaskType");

            var workflow = new Workflow(new WorkflowName("TestWorkflow"));
            workflow.AddTask(t1, new Payload("Node1"), type1, SimplePriority, EmptyTaskList, new[] { t2 });
            workflow.AddTask(t2, new Payload("Node2"), type1, SimplePriority, EmptyTaskList, new[] { t3, t4, t5, t6 });
            workflow.AddTask(t3, new Payload("Node3"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t4, new Payload("Node4"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t5, new Payload("Node5"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t6, new Payload("Node6"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), new[] { type1 }, new Lua()))
            {
                var workflowId = wm.PushWorkflow(workflow);

                th.Gate.WaitOne();

                wm.PauseWorkflow(workflowId);

                wm.ReleaseWorkflow(workflowId);

                Assert.AreEqual(1, db.ListLength("submitted:testTaskType"));
                Assert.AreEqual(1, db.SetLength("running"));
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
Ejemplo n.º 14
0
        public void CanResetOwnTasks()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler("1");
            var th2 = new BlockingTaskHandler("2");

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var t1 = new TaskName("TestNode1");
            var t2 = new TaskName("TestNode2");
            var t3 = new TaskName("TestNode3");
            var t4 = new TaskName("TestNode4");
            var t5 = new TaskName("TestNode5");
            var t6 = new TaskName("TestNode6");

            var workflow = new Workflow(new WorkflowName("TestWorkflow"));
            workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
            workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
            workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                wm.PushWorkflow(workflow);

                // we've picked up the first task, and don't want to pick up any more
                th.Gate.WaitOne();

                Assert.AreEqual(0, db.ListLength("submitted"));
            }

            // create a new wm to simulate a restart of a component
            using (var wm2 = new WorkflowManagement(_mux, th2, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                Assert.AreEqual(1, db.SortedSetLength("submitted"));

                th.Abort.Set();
                th2.LetRun.Set();

                complete.WaitOne();
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
Ejemplo n.º 15
0
        public void CanSubmitAWorkflowAsync()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler();
            th.LetRun.Set();

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");
                var t5 = new TaskName("TestNode5");
                var t6 = new TaskName("TestNode6");

                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
                workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

                var id = wm.PushWorkflowAsync(workflow);

                id.Wait();
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
Ejemplo n.º 16
0
        public void IfATaskIsMarkedFailed_WorfklowFailsAndNoMoreTasksSubmitted()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"IfATaskIsMarkedFailed_WorfklowFailsAndNoMoreTasksSubmitted\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new ErroringTestTaskHandler(2);

            var complete = new ManualResetEvent(false);
            var failed = new ManualResetEvent(false);

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { complete.Set(); };
            wh.WorkflowFailed += (s, w) => { failed.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");

                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4 });
                workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

                wm.PushWorkflow(workflow);

                var result = WaitHandle.WaitAny(new[] { failed, complete }, 2000);

                Assert.AreEqual(0, result);
                Assert.AreEqual(2, th.TaskRunCount); // only 2 get sent through the task handler; second fails, so no children are executed
            }
        }
Ejemplo n.º 17
0
        public void CanCleanUpAfterAWorkflow()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanCleanUpAfterAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new TestTaskHandler();

            var complete = new ManualResetEvent(false);

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");
                var t5 = new TaskName("TestNode5");
                var t6 = new TaskName("TestNode6");

                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
                workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

                var workflowId = wm.PushWorkflow(workflow);

                var workflowCompleted = complete.WaitOne(2000); // machine-performance dependent, but 2 seconds is a long time

                Assert.IsTrue(workflowCompleted);

                var info = wm.FetchWorkflowInformation(workflowId.ToString());

                Assert.AreEqual("1", info.Id);
                Assert.AreEqual(6, info.Tasks.Count);

                wm.CleanUp(workflowId.ToString());

                db = _mux.GetDatabase();
                for (var t = 0; t < 6; t++)
                {
                    Assert.IsFalse(db.KeyExists("task:" + t));
                    Assert.IsFalse(db.KeyExists("parents-" + t));
                    Assert.IsFalse(db.KeyExists("children-" + t));
                    Assert.IsFalse(db.SetContains("tasks", t));
                    Assert.IsFalse(db.SetContains("submitted", t));
                    Assert.IsFalse(db.SetContains("complete", t));
                    Assert.IsFalse(db.SetContains("failed", t));
                    Assert.IsFalse(db.SetContains("abandoned", t));
                    Assert.AreEqual(0, db.ListRemove("running", t));
                }
                Assert.AreEqual(0, db.ListRemove("workflowComplete", "1"));
                Assert.AreEqual(0, db.ListRemove("workflowFailed", "1"));
                Assert.IsFalse(db.KeyExists("workflow-tasks-1"));
                Assert.IsFalse(db.KeyExists("workflow-remaining-1"));
                Assert.IsFalse(db.SetContains("workflows", "1"));
                Assert.IsFalse(db.KeyExists("submitted:1"));
                Assert.IsFalse(db.KeyExists("running:1"));
            }
        }
Ejemplo n.º 18
0
        public void CanSubmitAndRunAWorkflowWithTypedTasks()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new TestTaskHandler();

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var type1 = new TaskType("testTaskType");
            var type2 = new TaskType("testTaskType2");

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), new[] { type1 }, new Lua()))
            using (var wm2 = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test2"), new[] { type2 }, new Lua()))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");
                var t5 = new TaskName("TestNode5");
                var t6 = new TaskName("TestNode6");
                
                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), type1, SimplePriority, EmptyTaskList, new[] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), type1, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
                workflow.AddTask(t3, new Payload("Node3"), type2, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t5, new Payload("Node5"), type2, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t6, new Payload("Node6"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);

                wm.PushWorkflow(workflow);

                var workflowWasCompleted = complete.WaitOne(2000);

                Assert.IsTrue(workflowWasCompleted);

                Console.WriteLine("WM events"); foreach (var ev in events) Console.WriteLine("Event: " + ev);

                Assert.AreEqual(6, th.TaskRunCount);
            }
        }
Ejemplo n.º 19
0
        public void CanCleanUpAfterAFailedWorkflow()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanCleanUpAfterAFailedWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new ErroringTestWithLongRunnerTaskHandler("Node4", "Node3");

            var complete = new ManualResetEvent(false);
            var failed = new ManualResetEvent(false);
            var exception = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
            wh.WorkflowFailed += (s, w) => { events.Add("failed"); failed.Set(); };

            EventHandler<Exception> eh = (s, e) => { events.Add(e.Message); exception.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), eh))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");

                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new [] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new [] { t1 }, new [] { t3, t4 });
                workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);

                var workflowId = wm.PushWorkflow(workflow);

                var result = WaitHandle.WaitAny(new[] { exception, failed, complete }, 2000);

                if(result == 0) throw new Exception("Failed in workflow operation"); // in a real app, handler could add exception to a list to process

                // In principle this is a failed workflow, as one of the tasks will fail
                // However, if all tasks get started, then the succeeding leaf node finishes first, this will be marked successful
                // Is a workflow considered complete when all tasks have started, or when all tasks have finished?
                // CHoose latter.
                Console.WriteLine("TaskHandler events"); foreach (var ev in th.Events) Console.WriteLine("TH event: " + ev);
                Console.WriteLine("WM events"); foreach (var ev in events) Console.WriteLine("Event: " + ev);
                Assert.AreEqual(1, result);

                wm.CleanUp(workflowId.ToString());
            }

            // Should leave highwatermark id keys alone
            db = _mux.GetDatabase();
            for (var t = 0; t < 4; t++)
            {
                Assert.IsFalse(db.KeyExists("task:" + t));
                Assert.IsFalse(db.KeyExists("parents-" + t));
                Assert.IsFalse(db.KeyExists("children-" + t));
                Assert.IsFalse(db.SetContains("tasks", t));
                Assert.IsFalse(db.SetContains("submitted", t));
                Assert.IsFalse(db.SetContains("complete", t));
                Assert.IsFalse(db.SetContains("failed", t));
                if (t == 3)
                {
                    Assert.IsTrue(db.SetContains("abandoned", t));
                    db.SetRemove("abandoned", t);
                }
                else
                    Assert.IsFalse(db.SetContains("abandoned", t));
                Assert.AreEqual(false, db.SetRemove("running", t));
            }
            Assert.AreEqual(0, db.ListRemove("workflowComplete", "1"));
            Assert.AreEqual(0, db.ListRemove("workflowFailed", "1"));
            Assert.IsFalse(db.KeyExists("workflow-tasks-1"));
            Assert.IsFalse(db.KeyExists("workflow-remaining-1"));
            Assert.IsFalse(db.SetContains("workflows", "1"));
        }
Ejemplo n.º 20
0
        public void HandlesExceptionsFromBadLuaInAFailingWorkflow()
        {
            var cases = new[] { TestCase.FailTask, TestCase.PopFailedWorkflow };

            foreach (var testCase in cases)
            {
                var lua = new BadLua(testCase);

                var message = "HandlesExceptionsFromBadLuaInAFailingWorkflow:" + testCase;

                var db = _mux.GetDatabase();
                db.ScriptEvaluate("print(\"" + message + "\")");
                db.ScriptEvaluate("redis.call(\"flushdb\")");

                var th = new ErroringTestWithLongRunnerTaskHandler("Node4", "Node3");

                var complete = new ManualResetEvent(false);
                var failed = new ManualResetEvent(false);
                var exception = new ManualResetEvent(false);

                var events = new List<string>();

                var wh = new WorkflowHandler();
                wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
                wh.WorkflowFailed += (s, w) => { events.Add("failed"); failed.Set(); };

                EventHandler<Exception> eh = (s, e) => { events.Add(e.Message); exception.Set(); };

                using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, lua, eh, Behaviours.Processor | Behaviours.Submitter))
                {
                    var t1 = new TaskName("TestNode1");
                    var t2 = new TaskName("TestNode2");
                    var t3 = new TaskName("TestNode3");
                    var t4 = new TaskName("TestNode4");

                    var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                    workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new [] { t2 });
                    workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new [] { t1 }, new [] { t3, t4 });
                    workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
                    workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
                    
                    wm.PushWorkflow(workflow);

                    var waitResult = WaitHandle.WaitAny(new[] { exception, failed, complete }, 2000);

                    Assert.AreEqual(0, waitResult);
                }
            }
        }
Ejemplo n.º 21
0
        public void CanCleanUpAfterAFailedWorkflow()
        {
            var db = _mux.GetDatabase();

            db.ScriptEvaluate("print(\"CanCleanUpAfterAFailedWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new ErroringTestWithLongRunnerTaskHandler("Node4", "Node3");

            var complete  = new ManualResetEvent(false);
            var failed    = new ManualResetEvent(false);
            var exception = new ManualResetEvent(false);

            var events = new List <string>();

            var wh = new WorkflowHandler();

            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
            wh.WorkflowFailed   += (s, w) => { events.Add("failed"); failed.Set(); };

            EventHandler <Exception> eh = (s, e) => { events.Add(e.Message); exception.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), eh))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");

                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new [] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new [] { t1 }, new [] { t3, t4 });
                workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);

                var workflowId = wm.PushWorkflow(workflow);

                var result = WaitHandle.WaitAny(new[] { exception, failed, complete }, 2000);

                if (result == 0)
                {
                    throw new Exception("Failed in workflow operation");             // in a real app, handler could add exception to a list to process
                }
                // In principle this is a failed workflow, as one of the tasks will fail
                // However, if all tasks get started, then the succeeding leaf node finishes first, this will be marked successful
                // Is a workflow considered complete when all tasks have started, or when all tasks have finished?
                // CHoose latter.
                Console.WriteLine("TaskHandler events"); foreach (var ev in th.Events)
                {
                    Console.WriteLine("TH event: " + ev);
                }
                Console.WriteLine("WM events"); foreach (var ev in events)
                {
                    Console.WriteLine("Event: " + ev);
                }
                Assert.AreEqual(1, result);

                wm.CleanUp(workflowId.ToString());
            }

            // Should leave highwatermark id keys alone
            db = _mux.GetDatabase();
            for (var t = 0; t < 4; t++)
            {
                Assert.IsFalse(db.KeyExists("task:" + t));
                Assert.IsFalse(db.KeyExists("parents-" + t));
                Assert.IsFalse(db.KeyExists("children-" + t));
                Assert.IsFalse(db.SetContains("tasks", t));
                Assert.IsFalse(db.SetContains("submitted", t));
                Assert.IsFalse(db.SetContains("complete", t));
                Assert.IsFalse(db.SetContains("failed", t));
                if (t == 3)
                {
                    Assert.IsTrue(db.SetContains("abandoned", t));
                    db.SetRemove("abandoned", t);
                }
                else
                {
                    Assert.IsFalse(db.SetContains("abandoned", t));
                }
                Assert.AreEqual(false, db.SetRemove("running", t));
            }
            Assert.AreEqual(0, db.ListRemove("workflowComplete", "1"));
            Assert.AreEqual(0, db.ListRemove("workflowFailed", "1"));
            Assert.IsFalse(db.KeyExists("workflow-tasks-1"));
            Assert.IsFalse(db.KeyExists("workflow-remaining-1"));
            Assert.IsFalse(db.SetContains("workflows", "1"));
        }
Ejemplo n.º 22
0
        public void CanCleanUpAfterAWorkflow()
        {
            var db = _mux.GetDatabase();

            db.ScriptEvaluate("print(\"CanCleanUpAfterAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new TestTaskHandler();

            var complete = new ManualResetEvent(false);

            var wh = new WorkflowHandler();

            wh.WorkflowComplete += (s, w) => { complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");
                var t5 = new TaskName("TestNode5");
                var t6 = new TaskName("TestNode6");

                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
                workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

                var workflowId = wm.PushWorkflow(workflow);

                var workflowCompleted = complete.WaitOne(2000); // machine-performance dependent, but 2 seconds is a long time

                Assert.IsTrue(workflowCompleted);

                var info = wm.FetchWorkflowInformation(workflowId.ToString());

                Assert.AreEqual("1", info.Id);
                Assert.AreEqual(6, info.Tasks.Count);

                wm.CleanUp(workflowId.ToString());

                db = _mux.GetDatabase();
                for (var t = 0; t < 6; t++)
                {
                    Assert.IsFalse(db.KeyExists("task:" + t));
                    Assert.IsFalse(db.KeyExists("parents-" + t));
                    Assert.IsFalse(db.KeyExists("children-" + t));
                    Assert.IsFalse(db.SetContains("tasks", t));
                    Assert.IsFalse(db.SetContains("submitted", t));
                    Assert.IsFalse(db.SetContains("complete", t));
                    Assert.IsFalse(db.SetContains("failed", t));
                    Assert.IsFalse(db.SetContains("abandoned", t));
                    Assert.AreEqual(0, db.ListRemove("running", t));
                }
                Assert.AreEqual(0, db.ListRemove("workflowComplete", "1"));
                Assert.AreEqual(0, db.ListRemove("workflowFailed", "1"));
                Assert.IsFalse(db.KeyExists("workflow-tasks-1"));
                Assert.IsFalse(db.KeyExists("workflow-remaining-1"));
                Assert.IsFalse(db.SetContains("workflows", "1"));
                Assert.IsFalse(db.KeyExists("submitted:1"));
                Assert.IsFalse(db.KeyExists("running:1"));
            }
        }
Ejemplo n.º 23
0
        public void CanPauseAWorkflowWithTypedTasks()
        {
            var db = _mux.GetDatabase();

            db.ScriptEvaluate("print(\"CanPauseAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler("1");

            var complete = new ManualResetEvent(false);

            var events = new List <string>();

            var wh = new WorkflowHandler();

            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var t1 = new TaskName("TestNode1");
            var t2 = new TaskName("TestNode2");
            var t3 = new TaskName("TestNode3");
            var t4 = new TaskName("TestNode4");
            var t5 = new TaskName("TestNode5");
            var t6 = new TaskName("TestNode6");

            var type1 = new TaskType("testTaskType");

            var workflow = new Workflow(new WorkflowName("TestWorkflow"));

            workflow.AddTask(t1, new Payload("Node1"), type1, SimplePriority, EmptyTaskList, new[] { t2 });
            workflow.AddTask(t2, new Payload("Node2"), type1, SimplePriority, EmptyTaskList, new[] { t3, t4, t5, t6 });
            workflow.AddTask(t3, new Payload("Node3"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t4, new Payload("Node4"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t5, new Payload("Node5"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t6, new Payload("Node6"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), new[] { type1 }, new Lua()))
            {
                var workflowId = wm.PushWorkflow(workflow);

                th.Gate.WaitOne();

                Assert.AreEqual(1, db.SortedSetLength("submitted:testTaskType"));
                Assert.AreEqual(1, db.SetLength("running"));

                wm.PauseWorkflow(workflowId);

                var expected = new List <string> {
                    "1", "2"
                };
                var state = "paused";

                CheckSetContent(db, expected, state);

                // The running task will run bang into the new state
                th.LetRun.Set();

                Thread.Sleep(500); // gah, honestly. I won't get a signal because no new tasks should be submitted

                expected = new List <string> {
                    "2"
                };
                state = "paused";

                CheckSetContent(db, expected, state);

                expected = new List <string> {
                    "2"
                };
                state = "running";

                CheckSetContent(db, expected, state, false);

                // If this is checked before the task has completed then there'll be two paused tasks
                // We've got no hook into the completion event as yet
                Assert.AreEqual(1, db.SetLength("paused"));
                Assert.AreEqual(1, db.SetLength("paused:1"));
                Assert.AreEqual(0, db.ListLength("submitted:testTaskType"));
                Assert.AreEqual(0, db.SetLength("running"));
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }