コード例 #1
0
        internal WorkflowManagement(ConnectionMultiplexer mux, ITaskHandler taskHandler, WorkflowHandler workflowHandler, string identifier, IEnumerable<string> typesProcessed, ILua lua, EventHandler<Exception> exceptionHandler = null, Behaviours behaviours = Behaviours.All)
        {
            _taskHandler = taskHandler;

            _workflowHandler = workflowHandler;

            if (exceptionHandler != null)
            {
                ExceptionThrown += exceptionHandler;
            }

            _typesProcessed = typesProcessed;

            _db = mux.GetDatabase();

            _sub = mux.GetSubscriber();

            if (_typesProcessed == null || _typesProcessed.Count() == 0)
            {
                _sub.Subscribe("submittedTask", (c, v) =>
                {
                    ProcessNextTask();
                });
            }
            else
            {
                foreach(var t in _typesProcessed)
                {
                    _sub.Subscribe("submittedTask:" + t, (c, v) =>
                    {
                        ProcessNextTask(t);
                    });
                }
            }

            _sub.Subscribe("workflowFailed", (c, v) =>
            {
                ProcessNextFailedWorkflow();
            });

            _sub.Subscribe("workflowComplete", (c, v) =>
            {
                ProcessNextCompleteWorkflow();
            });

            _lua = lua;
            _lua.LoadScripts(_db, mux.GetServer("localhost:6379"));

            _identifier = identifier;

            if (behaviours.HasFlag(Behaviours.AutoRestart))
            {
                var resubmittedTasks = ResubmitTasks();

                foreach (var item in resubmittedTasks)
                {
                    Console.WriteLine("Resubmitted {0}", item);
                }
            }
        }
コード例 #2
0
ファイル: Program.cs プロジェクト: timbarrass/redis.workflow
        internal void Run(string[] args)
        {
            // Should you want to decompose to running single tasks you can just create a workflow containing nodes with no parents or
            // children. The system works out where to start by identifying nodes with no parents, and assuming they're immediately
            // runnable. The hosting workflow then gives you a handle by which you can treat the individual tasks as a related set.
            var t1 = new TaskName("TestNode1");
            var t2 = new TaskName("TestNode2");
            var t3 = new TaskName("TestNode3");
            var t4 = new TaskName("TestNode4");
            var t5 = new TaskName("TestNode5");
            var t6 = new TaskName("TestNode6");

            var type1 = new TaskType("testTaskType");

            var workflow = new Common.Workflow(new WorkflowName("TestWorkflow"));
            workflow.AddTask(t1, new Payload("Node1"), type1, SimplePriority, EmptyTaskList, new[] { t2 });
            workflow.AddTask(t2, new Payload("Node2"), type1, SimplePriority, EmptyTaskList, new[] { t3, t4, t5, t6 });
            workflow.AddTask(t3, new Payload("Node3"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t4, new Payload("Node4"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t5, new Payload("Node5"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t6, new Payload("Node6"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);

            // A client will start acting on tasks immediately. May not want that to
            // be behaviour -- so might want to have an independent submit-only client. TaskHandler only needed for
            // submission. TaskHandler needs to be stateless, taking configuration for a specific task on a run call? No, that's
            // true per-task, but agiven ITaskHandler instance might be configured once wth e.g. compute resource connection details
            var th = new TaskHandler();
            var wh = new WorkflowHandler();
            var complete = new ManualResetEvent(false);
            var failed = new ManualResetEvent(false);
            wh.WorkflowComplete += (s, w) => { Console.WriteLine("workflow complete: " + w); complete.Set(); };
            wh.WorkflowFailed += (s, w) => { Console.WriteLine("workflow failed: " + w); failed.Set(); };

            EventHandler<Exception> eh = (s, e) => { };

            var wm = new WorkflowManagement(ConnectionMultiplexer.Connect("localhost"), th, wh, new WorkflowManagementId("sampleApp"), eh);

            wm.ClearBacklog();

            // Pushing a workflow causes it to be executed
            var id = wm.PushWorkflow(workflow);

            Console.WriteLine("Workflow pushed");

            WaitHandle.WaitAny(new[] { failed, complete });

            wm.CleanUp(id.ToString());

            Console.ReadLine();
        }
コード例 #3
0
        public void CanQueryForOwnTasks()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler();

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(new TaskName("TestNode1"), new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, EmptyTaskList);

                wm.PushWorkflow(workflow);

                var taskWasStarted = th.Gate.WaitOne(2000);

                Assert.IsTrue(taskWasStarted);

                var myTasks = wm.FindTasks();

                Assert.AreEqual(1, myTasks.Length);
                Assert.AreEqual("1", myTasks[0]);
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
コード例 #4
0
        public void IfATaskIsMarkedFailed_WorfklowFailsAndNoMoreTasksSubmitted()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"IfATaskIsMarkedFailed_WorfklowFailsAndNoMoreTasksSubmitted\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new ErroringTestTaskHandler(2);

            var complete = new ManualResetEvent(false);
            var failed = new ManualResetEvent(false);

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { complete.Set(); };
            wh.WorkflowFailed += (s, w) => { failed.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");

                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4 });
                workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

                wm.PushWorkflow(workflow);

                var result = WaitHandle.WaitAny(new[] { failed, complete }, 2000);

                Assert.AreEqual(0, result);
                Assert.AreEqual(2, th.TaskRunCount); // only 2 get sent through the task handler; second fails, so no children are executed
            }
        }
コード例 #5
0
        public void CanSubmitAndRunAWorkflowWithTypedTasks()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new TestTaskHandler();

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var type1 = new TaskType("testTaskType");
            var type2 = new TaskType("testTaskType2");

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), new[] { type1 }, new Lua()))
            using (var wm2 = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test2"), new[] { type2 }, new Lua()))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");
                var t5 = new TaskName("TestNode5");
                var t6 = new TaskName("TestNode6");
                
                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), type1, SimplePriority, EmptyTaskList, new[] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), type1, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
                workflow.AddTask(t3, new Payload("Node3"), type2, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t5, new Payload("Node5"), type2, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t6, new Payload("Node6"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);

                wm.PushWorkflow(workflow);

                var workflowWasCompleted = complete.WaitOne(2000);

                Assert.IsTrue(workflowWasCompleted);

                Console.WriteLine("WM events"); foreach (var ev in events) Console.WriteLine("Event: " + ev);

                Assert.AreEqual(6, th.TaskRunCount);
            }
        }
コード例 #6
0
        public void CanCleanUpAfterAFailedWorkflow()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanCleanUpAfterAFailedWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new ErroringTestWithLongRunnerTaskHandler("Node4", "Node3");

            var complete = new ManualResetEvent(false);
            var failed = new ManualResetEvent(false);
            var exception = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
            wh.WorkflowFailed += (s, w) => { events.Add("failed"); failed.Set(); };

            EventHandler<Exception> eh = (s, e) => { events.Add(e.Message); exception.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), eh))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");

                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new [] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new [] { t1 }, new [] { t3, t4 });
                workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);

                var workflowId = wm.PushWorkflow(workflow);

                var result = WaitHandle.WaitAny(new[] { exception, failed, complete }, 2000);

                if(result == 0) throw new Exception("Failed in workflow operation"); // in a real app, handler could add exception to a list to process

                // In principle this is a failed workflow, as one of the tasks will fail
                // However, if all tasks get started, then the succeeding leaf node finishes first, this will be marked successful
                // Is a workflow considered complete when all tasks have started, or when all tasks have finished?
                // CHoose latter.
                Console.WriteLine("TaskHandler events"); foreach (var ev in th.Events) Console.WriteLine("TH event: " + ev);
                Console.WriteLine("WM events"); foreach (var ev in events) Console.WriteLine("Event: " + ev);
                Assert.AreEqual(1, result);

                wm.CleanUp(workflowId.ToString());
            }

            // Should leave highwatermark id keys alone
            db = _mux.GetDatabase();
            for (var t = 0; t < 4; t++)
            {
                Assert.IsFalse(db.KeyExists("task:" + t));
                Assert.IsFalse(db.KeyExists("parents-" + t));
                Assert.IsFalse(db.KeyExists("children-" + t));
                Assert.IsFalse(db.SetContains("tasks", t));
                Assert.IsFalse(db.SetContains("submitted", t));
                Assert.IsFalse(db.SetContains("complete", t));
                Assert.IsFalse(db.SetContains("failed", t));
                if (t == 3)
                {
                    Assert.IsTrue(db.SetContains("abandoned", t));
                    db.SetRemove("abandoned", t);
                }
                else
                    Assert.IsFalse(db.SetContains("abandoned", t));
                Assert.AreEqual(false, db.SetRemove("running", t));
            }
            Assert.AreEqual(0, db.ListRemove("workflowComplete", "1"));
            Assert.AreEqual(0, db.ListRemove("workflowFailed", "1"));
            Assert.IsFalse(db.KeyExists("workflow-tasks-1"));
            Assert.IsFalse(db.KeyExists("workflow-remaining-1"));
            Assert.IsFalse(db.SetContains("workflows", "1"));
        }
コード例 #7
0
        public void HandlesExceptionsFromBadLuaInAFailingWorkflow()
        {
            var cases = new[] { TestCase.FailTask, TestCase.PopFailedWorkflow };

            foreach (var testCase in cases)
            {
                var lua = new BadLua(testCase);

                var message = "HandlesExceptionsFromBadLuaInAFailingWorkflow:" + testCase;

                var db = _mux.GetDatabase();
                db.ScriptEvaluate("print(\"" + message + "\")");
                db.ScriptEvaluate("redis.call(\"flushdb\")");

                var th = new ErroringTestWithLongRunnerTaskHandler("Node4", "Node3");

                var complete = new ManualResetEvent(false);
                var failed = new ManualResetEvent(false);
                var exception = new ManualResetEvent(false);

                var events = new List<string>();

                var wh = new WorkflowHandler();
                wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
                wh.WorkflowFailed += (s, w) => { events.Add("failed"); failed.Set(); };

                EventHandler<Exception> eh = (s, e) => { events.Add(e.Message); exception.Set(); };

                using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, lua, eh, Behaviours.Processor | Behaviours.Submitter))
                {
                    var t1 = new TaskName("TestNode1");
                    var t2 = new TaskName("TestNode2");
                    var t3 = new TaskName("TestNode3");
                    var t4 = new TaskName("TestNode4");

                    var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                    workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new [] { t2 });
                    workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new [] { t1 }, new [] { t3, t4 });
                    workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
                    workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
                    
                    wm.PushWorkflow(workflow);

                    var waitResult = WaitHandle.WaitAny(new[] { exception, failed, complete }, 2000);

                    Assert.AreEqual(0, waitResult);
                }
            }
        }
コード例 #8
0
        public void CanPauseAWorkflow()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanPauseAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler("1");

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var workflowName = "TestWorkflow";

            var tasks = new List<Task>();
            tasks.Add(new Task { Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
            tasks.Add(new Task { Type = "", Name = "TestNode2", Payload = "Node2", Parents = new string[] { }, Children = new string[] { "TestNode3", "TestNode4", "TestNode5", "TestNode6" }, Workflow = workflowName });
            tasks.Add(new Task { Type = "", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
            tasks.Add(new Task { Type = "", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
            tasks.Add(new Task { Type = "", Name = "TestNode5", Payload = "Node5", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
            tasks.Add(new Task { Type = "", Name = "TestNode6", Payload = "Node6", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });

            var workflow = new Workflow { Name = workflowName, Tasks = tasks };

            using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
            {
                var workflowId = wm.PushWorkflow(workflow);

                th.Gate.WaitOne();

                Assert.AreEqual(1, db.SortedSetLength("submitted"));
                Assert.AreEqual(1, db.SetLength("running"));

                wm.PauseWorkflow(workflowId);

                var expected = new List<string> { "1", "2" };
                var state = "paused";

                CheckSetContent(db, expected, state);

                // The running task will run bang into the new state
                th.LetRun.Set();

                Thread.Sleep(500); // gah, honestly. I won't get a signal because no new tasks should be submitted

                expected = new List<string> { "2" };
                state = "paused";

                CheckSetContent(db, expected, state);

                expected = new List<string> { "2" };
                state = "running";

                CheckSetContent(db, expected, state, false);

                // If this is checked before the task has completed then there'll be two paused tasks
                // We've got no hook into the completion event as yet
                Assert.AreEqual(1, db.SetLength("paused")); 
                Assert.AreEqual(1, db.SetLength("paused:1"));
                Assert.AreEqual(0, db.SortedSetLength("submitted"));
                Assert.AreEqual(0, db.SetLength("running"));
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
コード例 #9
0
        public void CanResetOwnTasks()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler("1");
            var th2 = new BlockingTaskHandler("2");

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };


            var workflowName = "TestWorkflow";

            var tasks = new List<Task>();
            tasks.Add(new Task { Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
            tasks.Add(new Task { Type = "", Name = "TestNode2", Payload = "Node2", Parents = new string[] { "TestNode1" }, Children = new string[] { "TestNode3", "TestNode4", "TestNode5", "TestNode6" }, Workflow = workflowName });
            tasks.Add(new Task { Type = "", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
            tasks.Add(new Task { Type = "", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
            tasks.Add(new Task { Type = "", Name = "TestNode5", Payload = "Node5", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
            tasks.Add(new Task { Type = "", Name = "TestNode6", Payload = "Node6", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });

            var workflow = new Workflow { Name = workflowName, Tasks = tasks };

            using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
            {
                wm.PushWorkflow(workflow);

                // we've picked up the first task, and don't want to pick up any more
                th.Gate.WaitOne();

                Assert.AreEqual(0, db.ListLength("submitted"));
            }

            // create a new wm to simulate a restart of a component
            using (var wm2 = new WorkflowManagement(_mux, th2, wh, "test", null, new Lua()))
            {
                Assert.AreEqual(1, db.SortedSetLength("submitted"));

                th.Abort.Set();
                th2.LetRun.Set();

                complete.WaitOne();
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
コード例 #10
0
 /// <summary>
 /// Instantiate a new workflow manager, able to submit workflows and process tasks.
 /// </summary>
 /// <param name="mux"></param>
 /// <param name="taskHandler"></param>
 /// <param name="workflowHandler"></param>
 /// <param name="identifier"></param>
 /// <param name="exceptionHandler"></param>
 /// <param name="lowestPriority">
 /// The lowest priority task this instance will consider. High priority is given a score of 0; lowest priority should be given 
 /// a score > 0. Defaults to 100. Note that you should have at least one instance able to cover the whole priority range you
 /// submit at.
 /// </param>
 /// <param name="typesProcessed"></param>
 /// <param name="behaviours"></param>
 public WorkflowManagement(ConnectionMultiplexer mux, ITaskHandler taskHandler, WorkflowHandler workflowHandler, string identifier, EventHandler<Exception> exceptionHandler, int lowestPriority = 100, IEnumerable<string> typesProcessed = null, Behaviours behaviours = Behaviours.All)
                 : this(mux, taskHandler, workflowHandler, identifier, typesProcessed, new Lua(lowestPriority), exceptionHandler, behaviours)
 {
     
 }
コード例 #11
0
        internal void Run(string[] args)
        {
            int workflows = 100000;
            bool pushDone = false;
            int completed = 0;
            var countLock = new object();
            var allDone = new ManualResetEvent(false);

            var completionEvents = new List<DateTime>();
            var submittedQueueLength = new List<Tuple<DateTime, string>>();
            var runningSetLength = new List<Tuple<DateTime, string>>();

            var mux = ConnectMultiplexer();

            var db = ConnectAndFlushDatabase(mux);

            var eh = CreateEventHandler(allDone);

            var th = CreateTaskHandler();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, id) => { lock (countLock) { completed++; Console.Write("pushDone: " + pushDone + " done: " + completed + "\r"); completionEvents.Add(DateTime.Now); if (completed == workflows) allDone.Set(); } };
            wh.WorkflowFailed += (s, id) => { lock (countLock) { completed++; Console.Write("pushDone: " + pushDone + " done: " + completed + "\r"); completionEvents.Add(DateTime.Now); if (completed == workflows) allDone.Set(); } };

            CreateQueueLengthSampler(submittedQueueLength, runningSetLength, db);

            var wm = BuildWorkflowManagers(mux, th, wh, eh);

            var workflow = CreateTestWorkflow();

            var startTime = DateTime.Now;

            for (int i = 0; i < workflows; i++)
            {
                var id = wm.PushWorkflow(workflow);
            }

            var pushDuration = new TimeSpan(DateTime.Now.Ticks - startTime.Ticks);
            pushDone = true;

            WaitHandle.WaitAny(new[] { allDone });

            var duration = new TimeSpan(DateTime.Now.Ticks - startTime.Ticks);
            Console.WriteLine("Done, {0} in {1} ms (push took {2} ms, {3} per second) tasks: {4} churn: {5} tasks per second", workflows, duration.TotalMilliseconds, pushDuration.TotalMilliseconds, workflows / pushDuration.TotalSeconds, workflows * 9, workflows * 9 / duration.TotalSeconds);

            Console.Write("Saving data .. ");

            ExportQueueLengths(submittedQueueLength, runningSetLength, startTime);

            ExportFinalWorkflowCompletion(completionEvents, startTime);

            ExportWorkflowCompletion(workflows, wm, startTime);

            ExportTaskData(workflows, wm, startTime);

            Console.WriteLine("done");

            Console.ReadLine();

            db.ScriptEvaluate("redis.call(\"flushdb\")"); // TODO need to look at timeouts
        }
コード例 #12
0
        private static WorkflowManagement BuildWorkflowManagers(ConnectionMultiplexer mux, TaskHandler th, WorkflowHandler wh, EventHandler<Exception> eh)
        {
            var wm = new WorkflowManagement(mux, th, wh, "sampleApp", eh);

            var extraProcessors = new List<WorkflowManagement>();
            for (int i = 0; i < 10; i++)
            {
                extraProcessors.Add(new WorkflowManagement(mux, th, wh, "processor-" + i, eh));
            }

            return wm;
        }
コード例 #13
0
 /// <summary>
 /// Instantiate a new workflow manager, able to submit workflows and process tasks.
 /// </summary>
 /// <param name="mux"></param>
 /// <param name="taskHandler"></param>
 /// <param name="workflowHandler"></param>
 /// <param name="identifier"></param>
 /// <param name="exceptionHandler"></param>
 /// <param name="lowestPriority">
 /// The lowest priority task this instance will consider. High priority is given a score of 0; lowest priority should be given
 /// a score > 0. Defaults to 100. Note that you should have at least one instance able to cover the whole priority range you
 /// submit at.
 /// </param>
 /// <param name="typesProcessed"></param>
 /// <param name="behaviours"></param>
 public WorkflowManagement(ConnectionMultiplexer mux, ITaskHandler taskHandler, WorkflowHandler workflowHandler, WorkflowManagementId identifier, EventHandler <Exception> exceptionHandler, int lowestPriority = 100, IEnumerable <TaskType> typesProcessed = null, Behaviours behaviours = Behaviours.All)
     : this(mux, taskHandler, workflowHandler, identifier, typesProcessed, new Lua(lowestPriority), exceptionHandler, behaviours)
 {
 }
コード例 #14
0
        internal WorkflowManagement(ConnectionMultiplexer mux, ITaskHandler taskHandler, WorkflowHandler workflowHandler, WorkflowManagementId identifier, IEnumerable <TaskType> typesProcessed, ILua lua, EventHandler <Exception> exceptionHandler = null, Behaviours behaviours = Behaviours.All)
        {
            _taskHandler = taskHandler;

            _workflowHandler = workflowHandler;

            if (exceptionHandler != null)
            {
                ExceptionThrown += exceptionHandler;
            }

            _typesProcessed = typesProcessed;

            _db = mux.GetDatabase();

            _sub = mux.GetSubscriber();

            if (_typesProcessed == null || _typesProcessed.Count() == 0)
            {
                _sub.Subscribe("submittedTask", (c, v) =>
                {
                    ProcessNextTask();
                });
            }
            else
            {
                foreach (var t in _typesProcessed.Select(ty => ty.ToString()))
                {
                    _sub.Subscribe("submittedTask:" + t, (c, v) =>
                    {
                        ProcessNextTask(t);
                    });
                }
            }

            _sub.Subscribe("workflowFailed", (c, v) =>
            {
                ProcessNextFailedWorkflow();
            });

            _sub.Subscribe("workflowComplete", (c, v) =>
            {
                ProcessNextCompleteWorkflow();
            });

            _lua = lua;
            _lua.LoadScripts(_db, mux.GetServer("localhost:6379"));

            _identifier = identifier;

            if (behaviours.HasFlag(Behaviours.AutoRestart))
            {
                var resubmittedTasks = ResubmitTasks();

                foreach (var item in resubmittedTasks)
                {
                    Console.WriteLine("Resubmitted {0}", item);
                }
            }
        }
コード例 #15
0
        public void CanPauseAWorkflowWithTypedTasks()
        {
            var db = _mux.GetDatabase();

            db.ScriptEvaluate("print(\"CanPauseAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler("1");

            var complete = new ManualResetEvent(false);

            var events = new List <string>();

            var wh = new WorkflowHandler();

            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var t1 = new TaskName("TestNode1");
            var t2 = new TaskName("TestNode2");
            var t3 = new TaskName("TestNode3");
            var t4 = new TaskName("TestNode4");
            var t5 = new TaskName("TestNode5");
            var t6 = new TaskName("TestNode6");

            var type1 = new TaskType("testTaskType");

            var workflow = new Workflow(new WorkflowName("TestWorkflow"));

            workflow.AddTask(t1, new Payload("Node1"), type1, SimplePriority, EmptyTaskList, new[] { t2 });
            workflow.AddTask(t2, new Payload("Node2"), type1, SimplePriority, EmptyTaskList, new[] { t3, t4, t5, t6 });
            workflow.AddTask(t3, new Payload("Node3"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t4, new Payload("Node4"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t5, new Payload("Node5"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t6, new Payload("Node6"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), new[] { type1 }, new Lua()))
            {
                var workflowId = wm.PushWorkflow(workflow);

                th.Gate.WaitOne();

                Assert.AreEqual(1, db.SortedSetLength("submitted:testTaskType"));
                Assert.AreEqual(1, db.SetLength("running"));

                wm.PauseWorkflow(workflowId);

                var expected = new List <string> {
                    "1", "2"
                };
                var state = "paused";

                CheckSetContent(db, expected, state);

                // The running task will run bang into the new state
                th.LetRun.Set();

                Thread.Sleep(500); // gah, honestly. I won't get a signal because no new tasks should be submitted

                expected = new List <string> {
                    "2"
                };
                state = "paused";

                CheckSetContent(db, expected, state);

                expected = new List <string> {
                    "2"
                };
                state = "running";

                CheckSetContent(db, expected, state, false);

                // If this is checked before the task has completed then there'll be two paused tasks
                // We've got no hook into the completion event as yet
                Assert.AreEqual(1, db.SetLength("paused"));
                Assert.AreEqual(1, db.SetLength("paused:1"));
                Assert.AreEqual(0, db.ListLength("submitted:testTaskType"));
                Assert.AreEqual(0, db.SetLength("running"));
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
コード例 #16
0
        public void CanReleaseAWorkflowWithTypedTasks()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanPauseAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler("1");

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var t1 = new TaskName("TestNode1");
            var t2 = new TaskName("TestNode2");
            var t3 = new TaskName("TestNode3");
            var t4 = new TaskName("TestNode4");
            var t5 = new TaskName("TestNode5");
            var t6 = new TaskName("TestNode6");

            var type1 = new TaskType("testTaskType");

            var workflow = new Workflow(new WorkflowName("TestWorkflow"));
            workflow.AddTask(t1, new Payload("Node1"), type1, SimplePriority, EmptyTaskList, new[] { t2 });
            workflow.AddTask(t2, new Payload("Node2"), type1, SimplePriority, EmptyTaskList, new[] { t3, t4, t5, t6 });
            workflow.AddTask(t3, new Payload("Node3"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t4, new Payload("Node4"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t5, new Payload("Node5"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t6, new Payload("Node6"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), new[] { type1 }, new Lua()))
            {
                var workflowId = wm.PushWorkflow(workflow);

                th.Gate.WaitOne();

                wm.PauseWorkflow(workflowId);

                wm.ReleaseWorkflow(workflowId);

                Assert.AreEqual(1, db.ListLength("submitted:testTaskType"));
                Assert.AreEqual(1, db.SetLength("running"));
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
コード例 #17
0
        public void CanCleanUpAfterAWorkflow()
        {
            var db = _mux.GetDatabase();

            db.ScriptEvaluate("print(\"CanCleanUpAfterAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new TestTaskHandler();

            var complete = new ManualResetEvent(false);

            var wh = new WorkflowHandler();

            wh.WorkflowComplete += (s, w) => { complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");
                var t5 = new TaskName("TestNode5");
                var t6 = new TaskName("TestNode6");

                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
                workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

                var workflowId = wm.PushWorkflow(workflow);

                var workflowCompleted = complete.WaitOne(2000); // machine-performance dependent, but 2 seconds is a long time

                Assert.IsTrue(workflowCompleted);

                var info = wm.FetchWorkflowInformation(workflowId.ToString());

                Assert.AreEqual("1", info.Id);
                Assert.AreEqual(6, info.Tasks.Count);

                wm.CleanUp(workflowId.ToString());

                db = _mux.GetDatabase();
                for (var t = 0; t < 6; t++)
                {
                    Assert.IsFalse(db.KeyExists("task:" + t));
                    Assert.IsFalse(db.KeyExists("parents-" + t));
                    Assert.IsFalse(db.KeyExists("children-" + t));
                    Assert.IsFalse(db.SetContains("tasks", t));
                    Assert.IsFalse(db.SetContains("submitted", t));
                    Assert.IsFalse(db.SetContains("complete", t));
                    Assert.IsFalse(db.SetContains("failed", t));
                    Assert.IsFalse(db.SetContains("abandoned", t));
                    Assert.AreEqual(0, db.ListRemove("running", t));
                }
                Assert.AreEqual(0, db.ListRemove("workflowComplete", "1"));
                Assert.AreEqual(0, db.ListRemove("workflowFailed", "1"));
                Assert.IsFalse(db.KeyExists("workflow-tasks-1"));
                Assert.IsFalse(db.KeyExists("workflow-remaining-1"));
                Assert.IsFalse(db.SetContains("workflows", "1"));
                Assert.IsFalse(db.KeyExists("submitted:1"));
                Assert.IsFalse(db.KeyExists("running:1"));
            }
        }
コード例 #18
0
        public void HandlesExceptionsFromBadLuaInAFailingWorkflow()
        {
            var cases = new[] { TestCase.FailTask, TestCase.PopFailedWorkflow };

            foreach (var testCase in cases)
            {
                var lua = new BadLua(testCase);

                var message = "HandlesExceptionsFromBadLuaInAFailingWorkflow:" + testCase;

                var db = _mux.GetDatabase();
                db.ScriptEvaluate("print(\"" + message + "\")");
                db.ScriptEvaluate("redis.call(\"flushdb\")");

                var th = new ErroringTestWithLongRunnerTaskHandler("Node4", "Node3");

                var complete = new ManualResetEvent(false);
                var failed = new ManualResetEvent(false);
                var exception = new ManualResetEvent(false);

                var events = new List<string>();

                var wh = new WorkflowHandler();
                wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
                wh.WorkflowFailed += (s, w) => { events.Add("failed"); failed.Set(); };

                EventHandler<Exception> eh = (s, e) => { events.Add(e.Message); exception.Set(); };

                using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, lua, eh, Behaviours.Processor | Behaviours.Submitter))
                {
                    var workflowName = "TestWorkflow";

                    var tasks = new List<Task>();
                    tasks.Add(new Task { Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
                    tasks.Add(new Task { Type = "", Name = "TestNode2", Payload = "Node2", Parents = new string[] { "TestNode1" }, Children = new string[] { "TestNode3", "TestNode4" }, Workflow = workflowName });
                    tasks.Add(new Task { Type = "", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
                    tasks.Add(new Task { Type = "", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });

                    var workflow = new Workflow { Name = workflowName, Tasks = tasks };

                    wm.PushWorkflow(workflow);

                    var waitResult = WaitHandle.WaitAny(new[] { exception, failed, complete }, 2000);

                    Assert.AreEqual(0, waitResult);
                }
            }
        }
コード例 #19
0
        public void CanReleaseAWorkflowWithTypedTasks()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanPauseAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler("1");

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var workflowName = "TestWorkflow";

            var tasks = new List<Task>();
            tasks.Add(new Task { Type = "testTaskType", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
            tasks.Add(new Task { Type = "testTaskType", Name = "TestNode2", Payload = "Node2", Parents = new string[] { }, Children = new string[] { "TestNode3", "TestNode4", "TestNode5", "TestNode6" }, Workflow = workflowName });
            tasks.Add(new Task { Type = "testTaskType", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
            tasks.Add(new Task { Type = "testTaskType", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
            tasks.Add(new Task { Type = "testTaskType", Name = "TestNode5", Payload = "Node5", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
            tasks.Add(new Task { Type = "testTaskType", Name = "TestNode6", Payload = "Node6", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });

            var workflow = new Workflow { Name = workflowName, Tasks = tasks };

            using (var wm = new WorkflowManagement(_mux, th, wh, "test", new[] { "testTaskType" }, new Lua()))
            {
                var workflowId = wm.PushWorkflow(workflow);

                th.Gate.WaitOne();

                wm.PauseWorkflow(workflowId);

                wm.ReleaseWorkflow(workflowId);

                Assert.AreEqual(1, db.ListLength("submitted:testTaskType"));
                Assert.AreEqual(1, db.SetLength("running"));
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
コード例 #20
0
        public void ThrowsIfAWorkflowHasNoRootParents()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"ThrowsIfAWorkflowHasNoRootParents\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler();
            th.LetRun.Set();

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
            {
                var workflowName = "TestWorkflow";

                var tasks = new List<Task>();
                tasks.Add(new Task { Name = "TestNode1", Payload = "Node1", Parents = new string[] { "TestNode0" }, Children = new string[] { }, Workflow = workflowName });

                var workflow = new Workflow { Name = workflowName, Tasks = tasks };

                wm.PushWorkflow(workflow);

                var result = WaitHandle.WaitAny(new[] { complete }, 2000);

                Assert.AreEqual(0, result);
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
コード例 #21
0
        public void HandlesExceptionsFromBadLuaInASucceedingWorkflow()
        {
            var cases = new[] { TestCase.PopTask, TestCase.CompleteTask, TestCase.PopCompleteWorkflow };

            foreach (var testCase in cases)
            {
                var lua = new BadLua(testCase);

                var message = "HandlesExceptionsFromBadLuaInASucceedingWorkflow:" + testCase;

                var db = _mux.GetDatabase();
                db.ScriptEvaluate("print(\"" + message + "\")");
                db.ScriptEvaluate("redis.call(\"flushdb\")");

                var th = new TestTaskHandler();

                var complete = new ManualResetEvent(false);
                var failed = new ManualResetEvent(false);
                var exception = new ManualResetEvent(false);

                var events = new List<string>();

                var wh = new WorkflowHandler();
                wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
                wh.WorkflowFailed += (s, w) => { events.Add("failed"); failed.Set(); };

                EventHandler<Exception> eh = (s, e) => { events.Add(e.Message); exception.Set(); };

                using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, lua, eh, Behaviours.Processor | Behaviours.Submitter))
                {
                    var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                    workflow.AddTask(new TaskName("TestNode1"), new Payload("Node1"), new TaskType(""), new TaskPriority(1), new TaskName[0], new TaskName[0]);

                    wm.PushWorkflow(workflow);

                    var waitResult = WaitHandle.WaitAny(new[] { exception, failed, complete }, 2000);

                    Assert.AreEqual(0, waitResult);
                }
            }
        }
コード例 #22
0
        public void RunsTasksInPriorityOrder()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new TestTaskHandler();

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
            {
                var workflowName = "TestWorkflow";

                // 6 independent tasks in this workflow
                var tasks = new List<Task>();
                tasks.Add(new Task { Priority = 3, Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Priority = 2, Type = "", Name = "TestNode2", Payload = "Node2", Parents = new string[] { }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Priority = 1, Type = "", Name = "TestNode3", Payload = "Node3", Parents = new string[] { }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Priority = 4, Type = "", Name = "TestNode4", Payload = "Node4", Parents = new string[] { }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Priority = 6, Type = "", Name = "TestNode5", Payload = "Node5", Parents = new string[] { }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Priority = 5, Type = "", Name = "TestNode6", Payload = "Node6", Parents = new string[] { }, Children = new string[] { }, Workflow = workflowName });

                var workflow = new Workflow { Name = workflowName, Tasks = tasks };

                wm.PushWorkflow(workflow);

                var workflowWasCompleted = complete.WaitOne(2000);

                Assert.IsTrue(workflowWasCompleted);

                Console.WriteLine("WM events"); foreach (var ev in events) Console.WriteLine("Event: " + ev);

                Assert.AreEqual(6, th.TaskRunCount);
                Assert.AreEqual("Node3", th.Payloads[0]);
                Assert.AreEqual("Node2", th.Payloads[1]);
                Assert.AreEqual("Node1", th.Payloads[2]);
                Assert.AreEqual("Node4", th.Payloads[3]);
                Assert.AreEqual("Node6", th.Payloads[4]);
                Assert.AreEqual("Node5", th.Payloads[5]);
            }
        }
コード例 #23
0
        public void ThrowsIfAWorkflowHasNoRootParents()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"ThrowsIfAWorkflowHasNoRootParents\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler();
            th.LetRun.Set();

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(new TaskName("TestNode1"), new Payload("Node1"), NoType, SimplePriority, new [] { new TaskName("SomeParent") }, EmptyTaskList);
                
                wm.PushWorkflow(workflow);

                var result = WaitHandle.WaitAny(new[] { complete }, 2000);

                Assert.AreEqual(0, result);
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
コード例 #24
0
        public void CanSubmitAndRunAWorkflowWithTypedTasks()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new TestTaskHandler();

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, "test", new[] { "testTaskType" }, new Lua()))
            using (var wm2 = new WorkflowManagement(_mux, th, wh, "test2", new[] { "testTaskType2" }, new Lua()))
            {
                var workflowName = "TestWorkflow";

                var tasks = new List<Task>();
                tasks.Add(new Task { Type = "testTaskType", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
                tasks.Add(new Task { Type = "testTaskType", Name = "TestNode2", Payload = "Node2", Parents = new string[] { "TestNode1" }, Children = new string[] { "TestNode3", "TestNode4", "TestNode5", "TestNode6" }, Workflow = workflowName });
                tasks.Add(new Task { Type = "testTaskType2", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Type = "testTaskType", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Type = "testTaskType2", Name = "TestNode5", Payload = "Node5", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Type = "testTaskType", Name = "TestNode6", Payload = "Node6", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });

                var workflow = new Workflow { Name = workflowName, Tasks = tasks };

                wm.PushWorkflow(workflow);

                var workflowWasCompleted = complete.WaitOne(2000);

                Assert.IsTrue(workflowWasCompleted);

                Console.WriteLine("WM events"); foreach (var ev in events) Console.WriteLine("Event: " + ev);

                Assert.AreEqual(6, th.TaskRunCount);
            }
        }
コード例 #25
0
        public void RunsTasksInPriorityOrder()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new TestTaskHandler();

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(new TaskName("TestNode1"), new Payload("Node1"), NoType, new TaskPriority(3), EmptyTaskList, EmptyTaskList);
                workflow.AddTask(new TaskName("TestNode2"), new Payload("Node2"), NoType, new TaskPriority(2), EmptyTaskList, EmptyTaskList);
                workflow.AddTask(new TaskName("TestNode3"), new Payload("Node3"), NoType, new TaskPriority(1), EmptyTaskList, EmptyTaskList);
                workflow.AddTask(new TaskName("TestNode4"), new Payload("Node4"), NoType, new TaskPriority(4), EmptyTaskList, EmptyTaskList);
                workflow.AddTask(new TaskName("TestNode5"), new Payload("Node5"), NoType, new TaskPriority(6), EmptyTaskList, EmptyTaskList);
                workflow.AddTask(new TaskName("TestNode6"), new Payload("Node6"), NoType, new TaskPriority(5), EmptyTaskList, EmptyTaskList);

                wm.PushWorkflow(workflow);

                var workflowWasCompleted = complete.WaitOne(2000);

                Assert.IsTrue(workflowWasCompleted);

                Console.WriteLine("WM events"); foreach (var ev in events) Console.WriteLine("Event: " + ev);

                Assert.AreEqual(6, th.TaskRunCount);
                Assert.AreEqual("Node3", th.Payloads[0]);
                Assert.AreEqual("Node2", th.Payloads[1]);
                Assert.AreEqual("Node1", th.Payloads[2]);
                Assert.AreEqual("Node4", th.Payloads[3]);
                Assert.AreEqual("Node6", th.Payloads[4]);
                Assert.AreEqual("Node5", th.Payloads[5]);
            }
        }
コード例 #26
0
        public void CanCleanUpAfterAWorkflow()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanCleanUpAfterAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new TestTaskHandler();

            var complete = new ManualResetEvent(false);

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
            {
                var workflowName = "TestWorkflow";

                var tasks = new List<Task>();
                tasks.Add(new Task { Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
                tasks.Add(new Task { Type = "", Name = "TestNode2", Payload = "Node2", Parents = new string[] { "TestNode1" }, Children = new string[] { "TestNode3", "TestNode4", "TestNode5", "TestNode6" }, Workflow = workflowName });
                tasks.Add(new Task { Type = "", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Type = "", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Type = "", Name = "TestNode5", Payload = "Node5", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Type = "", Name = "TestNode6", Payload = "Node6", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });

                var workflow = new Workflow { Name = workflowName, Tasks = tasks };

                var workflowId = wm.PushWorkflow(workflow);

                var workflowCompleted = complete.WaitOne(2000); // machine-performance dependent, but 2 seconds is a long time

                Assert.IsTrue(workflowCompleted);

                var info = wm.FetchWorkflowInformation(workflowId.ToString());

                Assert.AreEqual("1", info.Id);
                Assert.AreEqual(6, info.Tasks.Count);

                wm.CleanUp(workflowId.ToString());

                db = _mux.GetDatabase();
                for (var t = 0; t < 6; t++)
                {
                    Assert.IsFalse(db.KeyExists("task:" + t));
                    Assert.IsFalse(db.KeyExists("parents-" + t));
                    Assert.IsFalse(db.KeyExists("children-" + t));
                    Assert.IsFalse(db.SetContains("tasks", t));
                    Assert.IsFalse(db.SetContains("submitted", t));
                    Assert.IsFalse(db.SetContains("complete", t));
                    Assert.IsFalse(db.SetContains("failed", t));
                    Assert.IsFalse(db.SetContains("abandoned", t));
                    Assert.AreEqual(0, db.ListRemove("running", t));
                }
                Assert.AreEqual(0, db.ListRemove("workflowComplete", "1"));
                Assert.AreEqual(0, db.ListRemove("workflowFailed", "1"));
                Assert.IsFalse(db.KeyExists("workflow-tasks-1"));
                Assert.IsFalse(db.KeyExists("workflow-remaining-1"));
                Assert.IsFalse(db.SetContains("workflows", "1"));
                Assert.IsFalse(db.KeyExists("submitted:1"));
                Assert.IsFalse(db.KeyExists("running:1"));
            }
        }
コード例 #27
0
        public void CanCleanUpAfterAWorkflow()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanCleanUpAfterAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new TestTaskHandler();

            var complete = new ManualResetEvent(false);

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");
                var t5 = new TaskName("TestNode5");
                var t6 = new TaskName("TestNode6");

                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
                workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

                var workflowId = wm.PushWorkflow(workflow);

                var workflowCompleted = complete.WaitOne(2000); // machine-performance dependent, but 2 seconds is a long time

                Assert.IsTrue(workflowCompleted);

                var info = wm.FetchWorkflowInformation(workflowId.ToString());

                Assert.AreEqual("1", info.Id);
                Assert.AreEqual(6, info.Tasks.Count);

                wm.CleanUp(workflowId.ToString());

                db = _mux.GetDatabase();
                for (var t = 0; t < 6; t++)
                {
                    Assert.IsFalse(db.KeyExists("task:" + t));
                    Assert.IsFalse(db.KeyExists("parents-" + t));
                    Assert.IsFalse(db.KeyExists("children-" + t));
                    Assert.IsFalse(db.SetContains("tasks", t));
                    Assert.IsFalse(db.SetContains("submitted", t));
                    Assert.IsFalse(db.SetContains("complete", t));
                    Assert.IsFalse(db.SetContains("failed", t));
                    Assert.IsFalse(db.SetContains("abandoned", t));
                    Assert.AreEqual(0, db.ListRemove("running", t));
                }
                Assert.AreEqual(0, db.ListRemove("workflowComplete", "1"));
                Assert.AreEqual(0, db.ListRemove("workflowFailed", "1"));
                Assert.IsFalse(db.KeyExists("workflow-tasks-1"));
                Assert.IsFalse(db.KeyExists("workflow-remaining-1"));
                Assert.IsFalse(db.SetContains("workflows", "1"));
                Assert.IsFalse(db.KeyExists("submitted:1"));
                Assert.IsFalse(db.KeyExists("running:1"));
            }
        }
コード例 #28
0
        public void IfATaskIsMarkedFailed_WorfklowFailsAndNoMoreTasksSubmitted()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"IfATaskIsMarkedFailed_WorfklowFailsAndNoMoreTasksSubmitted\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new ErroringTestTaskHandler(2);

            var complete = new ManualResetEvent(false);
            var failed = new ManualResetEvent(false);

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { complete.Set(); };
            wh.WorkflowFailed += (s, w) => { failed.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
            {
                var workflowName = "TestWorkflow";

                var tasks = new List<Task>();
                tasks.Add(new Task { Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
                tasks.Add(new Task { Type = "", Name = "TestNode2", Payload = "Node2", Parents = new string[] { "TestNode1" }, Children = new string[] { "TestNode3", "TestNode4" }, Workflow = workflowName });
                tasks.Add(new Task { Type = "", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Type = "", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });

                var workflow = new Workflow { Name = workflowName, Tasks = tasks };

                wm.PushWorkflow(workflow);

                var result = WaitHandle.WaitAny(new[] { failed, complete }, 2000);

                Assert.AreEqual(0, result);
                Assert.AreEqual(2, th.TaskRunCount); // only 2 get sent through the task handler; second fails, so no children are executed
            }
        }
コード例 #29
0
        public void CanSubmitAWorkflowAsync()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler();
            th.LetRun.Set();

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");
                var t5 = new TaskName("TestNode5");
                var t6 = new TaskName("TestNode6");

                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
                workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
                workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

                var id = wm.PushWorkflowAsync(workflow);

                id.Wait();
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
コード例 #30
0
        public void CanSubmitAWorkflowAsync()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler();
            th.LetRun.Set();

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
            {
                var workflowName = "TestWorkflow";

                var tasks = new List<Task>();
                tasks.Add(new Task { Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
                tasks.Add(new Task { Type = "", Name = "TestNode2", Payload = "Node2", Parents = new string[] { "TestNode1" }, Children = new string[] { "TestNode3", "TestNode4", "TestNode5", "TestNode6" }, Workflow = workflowName });
                tasks.Add(new Task { Type = "", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Type = "", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Type = "", Name = "TestNode5", Payload = "Node5", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
                tasks.Add(new Task { Type = "", Name = "TestNode6", Payload = "Node6", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });

                var workflow = new Workflow { Name = workflowName, Tasks = tasks };

                var id = wm.PushWorkflowAsync(workflow);

                id.Wait();
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
コード例 #31
0
        public void CanResetOwnTasks()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler("1");
            var th2 = new BlockingTaskHandler("2");

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var t1 = new TaskName("TestNode1");
            var t2 = new TaskName("TestNode2");
            var t3 = new TaskName("TestNode3");
            var t4 = new TaskName("TestNode4");
            var t5 = new TaskName("TestNode5");
            var t6 = new TaskName("TestNode6");

            var workflow = new Workflow(new WorkflowName("TestWorkflow"));
            workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
            workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
            workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                wm.PushWorkflow(workflow);

                // we've picked up the first task, and don't want to pick up any more
                th.Gate.WaitOne();

                Assert.AreEqual(0, db.ListLength("submitted"));
            }

            // create a new wm to simulate a restart of a component
            using (var wm2 = new WorkflowManagement(_mux, th2, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                Assert.AreEqual(1, db.SortedSetLength("submitted"));

                th.Abort.Set();
                th2.LetRun.Set();

                complete.WaitOne();
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
コード例 #32
0
        public void CanQueryForOwnTasks()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler();

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
            {
                var workflowName = "TestWorkflow";

                var tasks = new List<Task>();
                tasks.Add(new Task { Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });

                var workflow = new Workflow { Name = workflowName, Tasks = tasks };

                wm.PushWorkflow(workflow);

                var taskWasStarted = th.Gate.WaitOne(2000);

                Assert.IsTrue(taskWasStarted);

                var myTasks = wm.FindTasks();

                Assert.AreEqual(1, myTasks.Length);
                Assert.AreEqual("1", myTasks[0]);
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
コード例 #33
0
        public void CanPauseAWorkflow()
        {
            var db = _mux.GetDatabase();
            db.ScriptEvaluate("print(\"CanPauseAWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new BlockingTaskHandler("1");

            var complete = new ManualResetEvent(false);

            var events = new List<string>();

            var wh = new WorkflowHandler();
            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };

            var t1 = new TaskName("TestNode1");
            var t2 = new TaskName("TestNode2");
            var t3 = new TaskName("TestNode3");
            var t4 = new TaskName("TestNode4");
            var t5 = new TaskName("TestNode5");
            var t6 = new TaskName("TestNode6");

            var type1 = new TaskType("testTaskType");

            var workflow = new Workflow(new WorkflowName("TestWorkflow"));
            workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
            workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, EmptyTaskList, new[] { t3, t4, t5, t6 });
            workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
            workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
            {
                var workflowId = wm.PushWorkflow(workflow);

                var waitResult = th.Gate.WaitOne(/*2000*/);

                Assert.IsTrue(waitResult);
                Assert.AreEqual(1, db.SortedSetLength("submitted"));
                Assert.AreEqual(1, db.SetLength("running"));

                wm.PauseWorkflow(workflowId);

                var expected = new List<string> { "1", "2" };
                var state = "paused";

                CheckSetContent(db, expected, state);

                // The running task will run bang into the new state
                th.LetRun.Set();

                Thread.Sleep(500); // gah, honestly. I won't get a signal because no new tasks should be submitted

                expected = new List<string> { "2" };
                state = "paused";

                CheckSetContent(db, expected, state);

                expected = new List<string> { "2" };
                state = "running";

                CheckSetContent(db, expected, state, false);

                // If this is checked before the task has completed then there'll be two paused tasks
                // We've got no hook into the completion event as yet
                Assert.AreEqual(1, db.SetLength("paused")); 
                Assert.AreEqual(1, db.SetLength("paused:1"));
                Assert.AreEqual(0, db.SortedSetLength("submitted"));
                Assert.AreEqual(0, db.SetLength("running"));
            }

            db.ScriptEvaluate("redis.call(\"flushdb\")");
        }
コード例 #34
0
        public void CanCleanUpAfterAFailedWorkflow()
        {
            var db = _mux.GetDatabase();

            db.ScriptEvaluate("print(\"CanCleanUpAfterAFailedWorkflow\")");
            db.ScriptEvaluate("redis.call(\"flushdb\")");

            var th = new ErroringTestWithLongRunnerTaskHandler("Node4", "Node3");

            var complete  = new ManualResetEvent(false);
            var failed    = new ManualResetEvent(false);
            var exception = new ManualResetEvent(false);

            var events = new List <string>();

            var wh = new WorkflowHandler();

            wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
            wh.WorkflowFailed   += (s, w) => { events.Add("failed"); failed.Set(); };

            EventHandler <Exception> eh = (s, e) => { events.Add(e.Message); exception.Set(); };

            using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), eh))
            {
                var t1 = new TaskName("TestNode1");
                var t2 = new TaskName("TestNode2");
                var t3 = new TaskName("TestNode3");
                var t4 = new TaskName("TestNode4");

                var workflow = new Workflow(new WorkflowName("TestWorkflow"));
                workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new [] { t2 });
                workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new [] { t1 }, new [] { t3, t4 });
                workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
                workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);

                var workflowId = wm.PushWorkflow(workflow);

                var result = WaitHandle.WaitAny(new[] { exception, failed, complete }, 2000);

                if (result == 0)
                {
                    throw new Exception("Failed in workflow operation");             // in a real app, handler could add exception to a list to process
                }
                // In principle this is a failed workflow, as one of the tasks will fail
                // However, if all tasks get started, then the succeeding leaf node finishes first, this will be marked successful
                // Is a workflow considered complete when all tasks have started, or when all tasks have finished?
                // CHoose latter.
                Console.WriteLine("TaskHandler events"); foreach (var ev in th.Events)
                {
                    Console.WriteLine("TH event: " + ev);
                }
                Console.WriteLine("WM events"); foreach (var ev in events)
                {
                    Console.WriteLine("Event: " + ev);
                }
                Assert.AreEqual(1, result);

                wm.CleanUp(workflowId.ToString());
            }

            // Should leave highwatermark id keys alone
            db = _mux.GetDatabase();
            for (var t = 0; t < 4; t++)
            {
                Assert.IsFalse(db.KeyExists("task:" + t));
                Assert.IsFalse(db.KeyExists("parents-" + t));
                Assert.IsFalse(db.KeyExists("children-" + t));
                Assert.IsFalse(db.SetContains("tasks", t));
                Assert.IsFalse(db.SetContains("submitted", t));
                Assert.IsFalse(db.SetContains("complete", t));
                Assert.IsFalse(db.SetContains("failed", t));
                if (t == 3)
                {
                    Assert.IsTrue(db.SetContains("abandoned", t));
                    db.SetRemove("abandoned", t);
                }
                else
                {
                    Assert.IsFalse(db.SetContains("abandoned", t));
                }
                Assert.AreEqual(false, db.SetRemove("running", t));
            }
            Assert.AreEqual(0, db.ListRemove("workflowComplete", "1"));
            Assert.AreEqual(0, db.ListRemove("workflowFailed", "1"));
            Assert.IsFalse(db.KeyExists("workflow-tasks-1"));
            Assert.IsFalse(db.KeyExists("workflow-remaining-1"));
            Assert.IsFalse(db.SetContains("workflows", "1"));
        }