internal WorkflowManagement(ConnectionMultiplexer mux, ITaskHandler taskHandler, WorkflowHandler workflowHandler, string identifier, IEnumerable<string> typesProcessed, ILua lua, EventHandler<Exception> exceptionHandler = null, Behaviours behaviours = Behaviours.All)
{
_taskHandler = taskHandler;
_workflowHandler = workflowHandler;
if (exceptionHandler != null)
{
ExceptionThrown += exceptionHandler;
}
_typesProcessed = typesProcessed;
_db = mux.GetDatabase();
_sub = mux.GetSubscriber();
if (_typesProcessed == null || _typesProcessed.Count() == 0)
{
_sub.Subscribe("submittedTask", (c, v) =>
{
ProcessNextTask();
});
}
else
{
foreach(var t in _typesProcessed)
{
_sub.Subscribe("submittedTask:" + t, (c, v) =>
{
ProcessNextTask(t);
});
}
}
_sub.Subscribe("workflowFailed", (c, v) =>
{
ProcessNextFailedWorkflow();
});
_sub.Subscribe("workflowComplete", (c, v) =>
{
ProcessNextCompleteWorkflow();
});
_lua = lua;
_lua.LoadScripts(_db, mux.GetServer("localhost:6379"));
_identifier = identifier;
if (behaviours.HasFlag(Behaviours.AutoRestart))
{
var resubmittedTasks = ResubmitTasks();
foreach (var item in resubmittedTasks)
{
Console.WriteLine("Resubmitted {0}", item);
}
}
}
internal void Run(string[] args)
{
// Should you want to decompose to running single tasks you can just create a workflow containing nodes with no parents or
// children. The system works out where to start by identifying nodes with no parents, and assuming they're immediately
// runnable. The hosting workflow then gives you a handle by which you can treat the individual tasks as a related set.
var t1 = new TaskName("TestNode1");
var t2 = new TaskName("TestNode2");
var t3 = new TaskName("TestNode3");
var t4 = new TaskName("TestNode4");
var t5 = new TaskName("TestNode5");
var t6 = new TaskName("TestNode6");
var type1 = new TaskType("testTaskType");
var workflow = new Common.Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(t1, new Payload("Node1"), type1, SimplePriority, EmptyTaskList, new[] { t2 });
workflow.AddTask(t2, new Payload("Node2"), type1, SimplePriority, EmptyTaskList, new[] { t3, t4, t5, t6 });
workflow.AddTask(t3, new Payload("Node3"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t4, new Payload("Node4"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t5, new Payload("Node5"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t6, new Payload("Node6"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
// A client will start acting on tasks immediately. May not want that to
// be behaviour -- so might want to have an independent submit-only client. TaskHandler only needed for
// submission. TaskHandler needs to be stateless, taking configuration for a specific task on a run call? No, that's
// true per-task, but agiven ITaskHandler instance might be configured once wth e.g. compute resource connection details
var th = new TaskHandler();
var wh = new WorkflowHandler();
var complete = new ManualResetEvent(false);
var failed = new ManualResetEvent(false);
wh.WorkflowComplete += (s, w) => { Console.WriteLine("workflow complete: " + w); complete.Set(); };
wh.WorkflowFailed += (s, w) => { Console.WriteLine("workflow failed: " + w); failed.Set(); };
EventHandler<Exception> eh = (s, e) => { };
var wm = new WorkflowManagement(ConnectionMultiplexer.Connect("localhost"), th, wh, new WorkflowManagementId("sampleApp"), eh);
wm.ClearBacklog();
// Pushing a workflow causes it to be executed
var id = wm.PushWorkflow(workflow);
Console.WriteLine("Workflow pushed");
WaitHandle.WaitAny(new[] { failed, complete });
wm.CleanUp(id.ToString());
Console.ReadLine();
}
public void CanQueryForOwnTasks()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new BlockingTaskHandler();
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
{
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(new TaskName("TestNode1"), new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, EmptyTaskList);
wm.PushWorkflow(workflow);
var taskWasStarted = th.Gate.WaitOne(2000);
Assert.IsTrue(taskWasStarted);
var myTasks = wm.FindTasks();
Assert.AreEqual(1, myTasks.Length);
Assert.AreEqual("1", myTasks[0]);
}
db.ScriptEvaluate("redis.call(\"flushdb\")");
}
public void IfATaskIsMarkedFailed_WorfklowFailsAndNoMoreTasksSubmitted()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"IfATaskIsMarkedFailed_WorfklowFailsAndNoMoreTasksSubmitted\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new ErroringTestTaskHandler(2);
var complete = new ManualResetEvent(false);
var failed = new ManualResetEvent(false);
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { complete.Set(); };
wh.WorkflowFailed += (s, w) => { failed.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
{
var t1 = new TaskName("TestNode1");
var t2 = new TaskName("TestNode2");
var t3 = new TaskName("TestNode3");
var t4 = new TaskName("TestNode4");
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4 });
workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
wm.PushWorkflow(workflow);
var result = WaitHandle.WaitAny(new[] { failed, complete }, 2000);
Assert.AreEqual(0, result);
Assert.AreEqual(2, th.TaskRunCount); // only 2 get sent through the task handler; second fails, so no children are executed
}
}
public void CanSubmitAndRunAWorkflowWithTypedTasks()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new TestTaskHandler();
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
var type1 = new TaskType("testTaskType");
var type2 = new TaskType("testTaskType2");
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), new[] { type1 }, new Lua()))
using (var wm2 = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test2"), new[] { type2 }, new Lua()))
{
var t1 = new TaskName("TestNode1");
var t2 = new TaskName("TestNode2");
var t3 = new TaskName("TestNode3");
var t4 = new TaskName("TestNode4");
var t5 = new TaskName("TestNode5");
var t6 = new TaskName("TestNode6");
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(t1, new Payload("Node1"), type1, SimplePriority, EmptyTaskList, new[] { t2 });
workflow.AddTask(t2, new Payload("Node2"), type1, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
workflow.AddTask(t3, new Payload("Node3"), type2, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t4, new Payload("Node4"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t5, new Payload("Node5"), type2, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t6, new Payload("Node6"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
wm.PushWorkflow(workflow);
var workflowWasCompleted = complete.WaitOne(2000);
Assert.IsTrue(workflowWasCompleted);
Console.WriteLine("WM events"); foreach (var ev in events) Console.WriteLine("Event: " + ev);
Assert.AreEqual(6, th.TaskRunCount);
}
}
public void CanCleanUpAfterAFailedWorkflow()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanCleanUpAfterAFailedWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new ErroringTestWithLongRunnerTaskHandler("Node4", "Node3");
var complete = new ManualResetEvent(false);
var failed = new ManualResetEvent(false);
var exception = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
wh.WorkflowFailed += (s, w) => { events.Add("failed"); failed.Set(); };
EventHandler<Exception> eh = (s, e) => { events.Add(e.Message); exception.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), eh))
{
var t1 = new TaskName("TestNode1");
var t2 = new TaskName("TestNode2");
var t3 = new TaskName("TestNode3");
var t4 = new TaskName("TestNode4");
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new [] { t2 });
workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new [] { t1 }, new [] { t3, t4 });
workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
var workflowId = wm.PushWorkflow(workflow);
var result = WaitHandle.WaitAny(new[] { exception, failed, complete }, 2000);
if(result == 0) throw new Exception("Failed in workflow operation"); // in a real app, handler could add exception to a list to process
// In principle this is a failed workflow, as one of the tasks will fail
// However, if all tasks get started, then the succeeding leaf node finishes first, this will be marked successful
// Is a workflow considered complete when all tasks have started, or when all tasks have finished?
// CHoose latter.
Console.WriteLine("TaskHandler events"); foreach (var ev in th.Events) Console.WriteLine("TH event: " + ev);
Console.WriteLine("WM events"); foreach (var ev in events) Console.WriteLine("Event: " + ev);
Assert.AreEqual(1, result);
wm.CleanUp(workflowId.ToString());
}
// Should leave highwatermark id keys alone
db = _mux.GetDatabase();
for (var t = 0; t < 4; t++)
{
Assert.IsFalse(db.KeyExists("task:" + t));
Assert.IsFalse(db.KeyExists("parents-" + t));
Assert.IsFalse(db.KeyExists("children-" + t));
Assert.IsFalse(db.SetContains("tasks", t));
Assert.IsFalse(db.SetContains("submitted", t));
Assert.IsFalse(db.SetContains("complete", t));
Assert.IsFalse(db.SetContains("failed", t));
if (t == 3)
{
Assert.IsTrue(db.SetContains("abandoned", t));
db.SetRemove("abandoned", t);
}
else
Assert.IsFalse(db.SetContains("abandoned", t));
Assert.AreEqual(false, db.SetRemove("running", t));
}
Assert.AreEqual(0, db.ListRemove("workflowComplete", "1"));
Assert.AreEqual(0, db.ListRemove("workflowFailed", "1"));
Assert.IsFalse(db.KeyExists("workflow-tasks-1"));
Assert.IsFalse(db.KeyExists("workflow-remaining-1"));
Assert.IsFalse(db.SetContains("workflows", "1"));
}
public void HandlesExceptionsFromBadLuaInAFailingWorkflow()
{
var cases = new[] { TestCase.FailTask, TestCase.PopFailedWorkflow };
foreach (var testCase in cases)
{
var lua = new BadLua(testCase);
var message = "HandlesExceptionsFromBadLuaInAFailingWorkflow:" + testCase;
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"" + message + "\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new ErroringTestWithLongRunnerTaskHandler("Node4", "Node3");
var complete = new ManualResetEvent(false);
var failed = new ManualResetEvent(false);
var exception = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
wh.WorkflowFailed += (s, w) => { events.Add("failed"); failed.Set(); };
EventHandler<Exception> eh = (s, e) => { events.Add(e.Message); exception.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, lua, eh, Behaviours.Processor | Behaviours.Submitter))
{
var t1 = new TaskName("TestNode1");
var t2 = new TaskName("TestNode2");
var t3 = new TaskName("TestNode3");
var t4 = new TaskName("TestNode4");
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new [] { t2 });
workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new [] { t1 }, new [] { t3, t4 });
workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
wm.PushWorkflow(workflow);
var waitResult = WaitHandle.WaitAny(new[] { exception, failed, complete }, 2000);
Assert.AreEqual(0, waitResult);
}
}
}
public void CanPauseAWorkflow()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanPauseAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new BlockingTaskHandler("1");
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
var workflowName = "TestWorkflow";
var tasks = new List<Task>();
tasks.Add(new Task { Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode2", Payload = "Node2", Parents = new string[] { }, Children = new string[] { "TestNode3", "TestNode4", "TestNode5", "TestNode6" }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode5", Payload = "Node5", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode6", Payload = "Node6", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
var workflow = new Workflow { Name = workflowName, Tasks = tasks };
using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
{
var workflowId = wm.PushWorkflow(workflow);
th.Gate.WaitOne();
Assert.AreEqual(1, db.SortedSetLength("submitted"));
Assert.AreEqual(1, db.SetLength("running"));
wm.PauseWorkflow(workflowId);
var expected = new List<string> { "1", "2" };
var state = "paused";
CheckSetContent(db, expected, state);
// The running task will run bang into the new state
th.LetRun.Set();
Thread.Sleep(500); // gah, honestly. I won't get a signal because no new tasks should be submitted
expected = new List<string> { "2" };
state = "paused";
CheckSetContent(db, expected, state);
expected = new List<string> { "2" };
state = "running";
CheckSetContent(db, expected, state, false);
// If this is checked before the task has completed then there'll be two paused tasks
// We've got no hook into the completion event as yet
Assert.AreEqual(1, db.SetLength("paused"));
Assert.AreEqual(1, db.SetLength("paused:1"));
Assert.AreEqual(0, db.SortedSetLength("submitted"));
Assert.AreEqual(0, db.SetLength("running"));
}
db.ScriptEvaluate("redis.call(\"flushdb\")");
}
public void CanResetOwnTasks()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new BlockingTaskHandler("1");
var th2 = new BlockingTaskHandler("2");
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
var workflowName = "TestWorkflow";
var tasks = new List<Task>();
tasks.Add(new Task { Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode2", Payload = "Node2", Parents = new string[] { "TestNode1" }, Children = new string[] { "TestNode3", "TestNode4", "TestNode5", "TestNode6" }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode5", Payload = "Node5", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode6", Payload = "Node6", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
var workflow = new Workflow { Name = workflowName, Tasks = tasks };
using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
{
wm.PushWorkflow(workflow);
// we've picked up the first task, and don't want to pick up any more
th.Gate.WaitOne();
Assert.AreEqual(0, db.ListLength("submitted"));
}
// create a new wm to simulate a restart of a component
using (var wm2 = new WorkflowManagement(_mux, th2, wh, "test", null, new Lua()))
{
Assert.AreEqual(1, db.SortedSetLength("submitted"));
th.Abort.Set();
th2.LetRun.Set();
complete.WaitOne();
}
db.ScriptEvaluate("redis.call(\"flushdb\")");
}
/// <summary>
/// Instantiate a new workflow manager, able to submit workflows and process tasks.
/// </summary>
/// <param name="mux"></param>
/// <param name="taskHandler"></param>
/// <param name="workflowHandler"></param>
/// <param name="identifier"></param>
/// <param name="exceptionHandler"></param>
/// <param name="lowestPriority">
/// The lowest priority task this instance will consider. High priority is given a score of 0; lowest priority should be given
/// a score > 0. Defaults to 100. Note that you should have at least one instance able to cover the whole priority range you
/// submit at.
/// </param>
/// <param name="typesProcessed"></param>
/// <param name="behaviours"></param>
public WorkflowManagement(ConnectionMultiplexer mux, ITaskHandler taskHandler, WorkflowHandler workflowHandler, string identifier, EventHandler<Exception> exceptionHandler, int lowestPriority = 100, IEnumerable<string> typesProcessed = null, Behaviours behaviours = Behaviours.All)
: this(mux, taskHandler, workflowHandler, identifier, typesProcessed, new Lua(lowestPriority), exceptionHandler, behaviours)
{
}
internal void Run(string[] args)
{
int workflows = 100000;
bool pushDone = false;
int completed = 0;
var countLock = new object();
var allDone = new ManualResetEvent(false);
var completionEvents = new List<DateTime>();
var submittedQueueLength = new List<Tuple<DateTime, string>>();
var runningSetLength = new List<Tuple<DateTime, string>>();
var mux = ConnectMultiplexer();
var db = ConnectAndFlushDatabase(mux);
var eh = CreateEventHandler(allDone);
var th = CreateTaskHandler();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, id) => { lock (countLock) { completed++; Console.Write("pushDone: " + pushDone + " done: " + completed + "\r"); completionEvents.Add(DateTime.Now); if (completed == workflows) allDone.Set(); } };
wh.WorkflowFailed += (s, id) => { lock (countLock) { completed++; Console.Write("pushDone: " + pushDone + " done: " + completed + "\r"); completionEvents.Add(DateTime.Now); if (completed == workflows) allDone.Set(); } };
CreateQueueLengthSampler(submittedQueueLength, runningSetLength, db);
var wm = BuildWorkflowManagers(mux, th, wh, eh);
var workflow = CreateTestWorkflow();
var startTime = DateTime.Now;
for (int i = 0; i < workflows; i++)
{
var id = wm.PushWorkflow(workflow);
}
var pushDuration = new TimeSpan(DateTime.Now.Ticks - startTime.Ticks);
pushDone = true;
WaitHandle.WaitAny(new[] { allDone });
var duration = new TimeSpan(DateTime.Now.Ticks - startTime.Ticks);
Console.WriteLine("Done, {0} in {1} ms (push took {2} ms, {3} per second) tasks: {4} churn: {5} tasks per second", workflows, duration.TotalMilliseconds, pushDuration.TotalMilliseconds, workflows / pushDuration.TotalSeconds, workflows * 9, workflows * 9 / duration.TotalSeconds);
Console.Write("Saving data .. ");
ExportQueueLengths(submittedQueueLength, runningSetLength, startTime);
ExportFinalWorkflowCompletion(completionEvents, startTime);
ExportWorkflowCompletion(workflows, wm, startTime);
ExportTaskData(workflows, wm, startTime);
Console.WriteLine("done");
Console.ReadLine();
db.ScriptEvaluate("redis.call(\"flushdb\")"); // TODO need to look at timeouts
}
private static WorkflowManagement BuildWorkflowManagers(ConnectionMultiplexer mux, TaskHandler th, WorkflowHandler wh, EventHandler<Exception> eh)
{
var wm = new WorkflowManagement(mux, th, wh, "sampleApp", eh);
var extraProcessors = new List<WorkflowManagement>();
for (int i = 0; i < 10; i++)
{
extraProcessors.Add(new WorkflowManagement(mux, th, wh, "processor-" + i, eh));
}
return wm;
}
/// <summary>
/// Instantiate a new workflow manager, able to submit workflows and process tasks.
/// </summary>
/// <param name="mux"></param>
/// <param name="taskHandler"></param>
/// <param name="workflowHandler"></param>
/// <param name="identifier"></param>
/// <param name="exceptionHandler"></param>
/// <param name="lowestPriority">
/// The lowest priority task this instance will consider. High priority is given a score of 0; lowest priority should be given
/// a score > 0. Defaults to 100. Note that you should have at least one instance able to cover the whole priority range you
/// submit at.
/// </param>
/// <param name="typesProcessed"></param>
/// <param name="behaviours"></param>
public WorkflowManagement(ConnectionMultiplexer mux, ITaskHandler taskHandler, WorkflowHandler workflowHandler, WorkflowManagementId identifier, EventHandler <Exception> exceptionHandler, int lowestPriority = 100, IEnumerable <TaskType> typesProcessed = null, Behaviours behaviours = Behaviours.All)
: this(mux, taskHandler, workflowHandler, identifier, typesProcessed, new Lua(lowestPriority), exceptionHandler, behaviours)
{
}
internal WorkflowManagement(ConnectionMultiplexer mux, ITaskHandler taskHandler, WorkflowHandler workflowHandler, WorkflowManagementId identifier, IEnumerable <TaskType> typesProcessed, ILua lua, EventHandler <Exception> exceptionHandler = null, Behaviours behaviours = Behaviours.All)
{
_taskHandler = taskHandler;
_workflowHandler = workflowHandler;
if (exceptionHandler != null)
{
ExceptionThrown += exceptionHandler;
}
_typesProcessed = typesProcessed;
_db = mux.GetDatabase();
_sub = mux.GetSubscriber();
if (_typesProcessed == null || _typesProcessed.Count() == 0)
{
_sub.Subscribe("submittedTask", (c, v) =>
{
ProcessNextTask();
});
}
else
{
foreach (var t in _typesProcessed.Select(ty => ty.ToString()))
{
_sub.Subscribe("submittedTask:" + t, (c, v) =>
{
ProcessNextTask(t);
});
}
}
_sub.Subscribe("workflowFailed", (c, v) =>
{
ProcessNextFailedWorkflow();
});
_sub.Subscribe("workflowComplete", (c, v) =>
{
ProcessNextCompleteWorkflow();
});
_lua = lua;
_lua.LoadScripts(_db, mux.GetServer("localhost:6379"));
_identifier = identifier;
if (behaviours.HasFlag(Behaviours.AutoRestart))
{
var resubmittedTasks = ResubmitTasks();
foreach (var item in resubmittedTasks)
{
Console.WriteLine("Resubmitted {0}", item);
}
}
}
public void CanPauseAWorkflowWithTypedTasks()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanPauseAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new BlockingTaskHandler("1");
var complete = new ManualResetEvent(false);
var events = new List <string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
var t1 = new TaskName("TestNode1");
var t2 = new TaskName("TestNode2");
var t3 = new TaskName("TestNode3");
var t4 = new TaskName("TestNode4");
var t5 = new TaskName("TestNode5");
var t6 = new TaskName("TestNode6");
var type1 = new TaskType("testTaskType");
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(t1, new Payload("Node1"), type1, SimplePriority, EmptyTaskList, new[] { t2 });
workflow.AddTask(t2, new Payload("Node2"), type1, SimplePriority, EmptyTaskList, new[] { t3, t4, t5, t6 });
workflow.AddTask(t3, new Payload("Node3"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t4, new Payload("Node4"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t5, new Payload("Node5"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t6, new Payload("Node6"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), new[] { type1 }, new Lua()))
{
var workflowId = wm.PushWorkflow(workflow);
th.Gate.WaitOne();
Assert.AreEqual(1, db.SortedSetLength("submitted:testTaskType"));
Assert.AreEqual(1, db.SetLength("running"));
wm.PauseWorkflow(workflowId);
var expected = new List <string> {
"1", "2"
};
var state = "paused";
CheckSetContent(db, expected, state);
// The running task will run bang into the new state
th.LetRun.Set();
Thread.Sleep(500); // gah, honestly. I won't get a signal because no new tasks should be submitted
expected = new List <string> {
"2"
};
state = "paused";
CheckSetContent(db, expected, state);
expected = new List <string> {
"2"
};
state = "running";
CheckSetContent(db, expected, state, false);
// If this is checked before the task has completed then there'll be two paused tasks
// We've got no hook into the completion event as yet
Assert.AreEqual(1, db.SetLength("paused"));
Assert.AreEqual(1, db.SetLength("paused:1"));
Assert.AreEqual(0, db.ListLength("submitted:testTaskType"));
Assert.AreEqual(0, db.SetLength("running"));
}
db.ScriptEvaluate("redis.call(\"flushdb\")");
}
public void CanReleaseAWorkflowWithTypedTasks()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanPauseAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new BlockingTaskHandler("1");
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
var t1 = new TaskName("TestNode1");
var t2 = new TaskName("TestNode2");
var t3 = new TaskName("TestNode3");
var t4 = new TaskName("TestNode4");
var t5 = new TaskName("TestNode5");
var t6 = new TaskName("TestNode6");
var type1 = new TaskType("testTaskType");
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(t1, new Payload("Node1"), type1, SimplePriority, EmptyTaskList, new[] { t2 });
workflow.AddTask(t2, new Payload("Node2"), type1, SimplePriority, EmptyTaskList, new[] { t3, t4, t5, t6 });
workflow.AddTask(t3, new Payload("Node3"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t4, new Payload("Node4"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t5, new Payload("Node5"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t6, new Payload("Node6"), type1, SimplePriority, new[] { t2 }, EmptyTaskList);
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), new[] { type1 }, new Lua()))
{
var workflowId = wm.PushWorkflow(workflow);
th.Gate.WaitOne();
wm.PauseWorkflow(workflowId);
wm.ReleaseWorkflow(workflowId);
Assert.AreEqual(1, db.ListLength("submitted:testTaskType"));
Assert.AreEqual(1, db.SetLength("running"));
}
db.ScriptEvaluate("redis.call(\"flushdb\")");
}
public void CanCleanUpAfterAWorkflow()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanCleanUpAfterAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new TestTaskHandler();
var complete = new ManualResetEvent(false);
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { complete.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
{
var t1 = new TaskName("TestNode1");
var t2 = new TaskName("TestNode2");
var t3 = new TaskName("TestNode3");
var t4 = new TaskName("TestNode4");
var t5 = new TaskName("TestNode5");
var t6 = new TaskName("TestNode6");
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
var workflowId = wm.PushWorkflow(workflow);
var workflowCompleted = complete.WaitOne(2000); // machine-performance dependent, but 2 seconds is a long time
Assert.IsTrue(workflowCompleted);
var info = wm.FetchWorkflowInformation(workflowId.ToString());
Assert.AreEqual("1", info.Id);
Assert.AreEqual(6, info.Tasks.Count);
wm.CleanUp(workflowId.ToString());
db = _mux.GetDatabase();
for (var t = 0; t < 6; t++)
{
Assert.IsFalse(db.KeyExists("task:" + t));
Assert.IsFalse(db.KeyExists("parents-" + t));
Assert.IsFalse(db.KeyExists("children-" + t));
Assert.IsFalse(db.SetContains("tasks", t));
Assert.IsFalse(db.SetContains("submitted", t));
Assert.IsFalse(db.SetContains("complete", t));
Assert.IsFalse(db.SetContains("failed", t));
Assert.IsFalse(db.SetContains("abandoned", t));
Assert.AreEqual(0, db.ListRemove("running", t));
}
Assert.AreEqual(0, db.ListRemove("workflowComplete", "1"));
Assert.AreEqual(0, db.ListRemove("workflowFailed", "1"));
Assert.IsFalse(db.KeyExists("workflow-tasks-1"));
Assert.IsFalse(db.KeyExists("workflow-remaining-1"));
Assert.IsFalse(db.SetContains("workflows", "1"));
Assert.IsFalse(db.KeyExists("submitted:1"));
Assert.IsFalse(db.KeyExists("running:1"));
}
}
public void HandlesExceptionsFromBadLuaInAFailingWorkflow()
{
var cases = new[] { TestCase.FailTask, TestCase.PopFailedWorkflow };
foreach (var testCase in cases)
{
var lua = new BadLua(testCase);
var message = "HandlesExceptionsFromBadLuaInAFailingWorkflow:" + testCase;
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"" + message + "\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new ErroringTestWithLongRunnerTaskHandler("Node4", "Node3");
var complete = new ManualResetEvent(false);
var failed = new ManualResetEvent(false);
var exception = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
wh.WorkflowFailed += (s, w) => { events.Add("failed"); failed.Set(); };
EventHandler<Exception> eh = (s, e) => { events.Add(e.Message); exception.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, lua, eh, Behaviours.Processor | Behaviours.Submitter))
{
var workflowName = "TestWorkflow";
var tasks = new List<Task>();
tasks.Add(new Task { Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode2", Payload = "Node2", Parents = new string[] { "TestNode1" }, Children = new string[] { "TestNode3", "TestNode4" }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
var workflow = new Workflow { Name = workflowName, Tasks = tasks };
wm.PushWorkflow(workflow);
var waitResult = WaitHandle.WaitAny(new[] { exception, failed, complete }, 2000);
Assert.AreEqual(0, waitResult);
}
}
}
public void CanReleaseAWorkflowWithTypedTasks()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanPauseAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new BlockingTaskHandler("1");
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
var workflowName = "TestWorkflow";
var tasks = new List<Task>();
tasks.Add(new Task { Type = "testTaskType", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
tasks.Add(new Task { Type = "testTaskType", Name = "TestNode2", Payload = "Node2", Parents = new string[] { }, Children = new string[] { "TestNode3", "TestNode4", "TestNode5", "TestNode6" }, Workflow = workflowName });
tasks.Add(new Task { Type = "testTaskType", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "testTaskType", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "testTaskType", Name = "TestNode5", Payload = "Node5", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "testTaskType", Name = "TestNode6", Payload = "Node6", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
var workflow = new Workflow { Name = workflowName, Tasks = tasks };
using (var wm = new WorkflowManagement(_mux, th, wh, "test", new[] { "testTaskType" }, new Lua()))
{
var workflowId = wm.PushWorkflow(workflow);
th.Gate.WaitOne();
wm.PauseWorkflow(workflowId);
wm.ReleaseWorkflow(workflowId);
Assert.AreEqual(1, db.ListLength("submitted:testTaskType"));
Assert.AreEqual(1, db.SetLength("running"));
}
db.ScriptEvaluate("redis.call(\"flushdb\")");
}
public void ThrowsIfAWorkflowHasNoRootParents()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"ThrowsIfAWorkflowHasNoRootParents\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new BlockingTaskHandler();
th.LetRun.Set();
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
{
var workflowName = "TestWorkflow";
var tasks = new List<Task>();
tasks.Add(new Task { Name = "TestNode1", Payload = "Node1", Parents = new string[] { "TestNode0" }, Children = new string[] { }, Workflow = workflowName });
var workflow = new Workflow { Name = workflowName, Tasks = tasks };
wm.PushWorkflow(workflow);
var result = WaitHandle.WaitAny(new[] { complete }, 2000);
Assert.AreEqual(0, result);
}
db.ScriptEvaluate("redis.call(\"flushdb\")");
}
public void HandlesExceptionsFromBadLuaInASucceedingWorkflow()
{
var cases = new[] { TestCase.PopTask, TestCase.CompleteTask, TestCase.PopCompleteWorkflow };
foreach (var testCase in cases)
{
var lua = new BadLua(testCase);
var message = "HandlesExceptionsFromBadLuaInASucceedingWorkflow:" + testCase;
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"" + message + "\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new TestTaskHandler();
var complete = new ManualResetEvent(false);
var failed = new ManualResetEvent(false);
var exception = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
wh.WorkflowFailed += (s, w) => { events.Add("failed"); failed.Set(); };
EventHandler<Exception> eh = (s, e) => { events.Add(e.Message); exception.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, lua, eh, Behaviours.Processor | Behaviours.Submitter))
{
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(new TaskName("TestNode1"), new Payload("Node1"), new TaskType(""), new TaskPriority(1), new TaskName[0], new TaskName[0]);
wm.PushWorkflow(workflow);
var waitResult = WaitHandle.WaitAny(new[] { exception, failed, complete }, 2000);
Assert.AreEqual(0, waitResult);
}
}
}
public void RunsTasksInPriorityOrder()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new TestTaskHandler();
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
{
var workflowName = "TestWorkflow";
// 6 independent tasks in this workflow
var tasks = new List<Task>();
tasks.Add(new Task { Priority = 3, Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Priority = 2, Type = "", Name = "TestNode2", Payload = "Node2", Parents = new string[] { }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Priority = 1, Type = "", Name = "TestNode3", Payload = "Node3", Parents = new string[] { }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Priority = 4, Type = "", Name = "TestNode4", Payload = "Node4", Parents = new string[] { }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Priority = 6, Type = "", Name = "TestNode5", Payload = "Node5", Parents = new string[] { }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Priority = 5, Type = "", Name = "TestNode6", Payload = "Node6", Parents = new string[] { }, Children = new string[] { }, Workflow = workflowName });
var workflow = new Workflow { Name = workflowName, Tasks = tasks };
wm.PushWorkflow(workflow);
var workflowWasCompleted = complete.WaitOne(2000);
Assert.IsTrue(workflowWasCompleted);
Console.WriteLine("WM events"); foreach (var ev in events) Console.WriteLine("Event: " + ev);
Assert.AreEqual(6, th.TaskRunCount);
Assert.AreEqual("Node3", th.Payloads[0]);
Assert.AreEqual("Node2", th.Payloads[1]);
Assert.AreEqual("Node1", th.Payloads[2]);
Assert.AreEqual("Node4", th.Payloads[3]);
Assert.AreEqual("Node6", th.Payloads[4]);
Assert.AreEqual("Node5", th.Payloads[5]);
}
}
public void ThrowsIfAWorkflowHasNoRootParents()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"ThrowsIfAWorkflowHasNoRootParents\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new BlockingTaskHandler();
th.LetRun.Set();
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
{
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(new TaskName("TestNode1"), new Payload("Node1"), NoType, SimplePriority, new [] { new TaskName("SomeParent") }, EmptyTaskList);
wm.PushWorkflow(workflow);
var result = WaitHandle.WaitAny(new[] { complete }, 2000);
Assert.AreEqual(0, result);
}
db.ScriptEvaluate("redis.call(\"flushdb\")");
}
public void CanSubmitAndRunAWorkflowWithTypedTasks()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new TestTaskHandler();
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, "test", new[] { "testTaskType" }, new Lua()))
using (var wm2 = new WorkflowManagement(_mux, th, wh, "test2", new[] { "testTaskType2" }, new Lua()))
{
var workflowName = "TestWorkflow";
var tasks = new List<Task>();
tasks.Add(new Task { Type = "testTaskType", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
tasks.Add(new Task { Type = "testTaskType", Name = "TestNode2", Payload = "Node2", Parents = new string[] { "TestNode1" }, Children = new string[] { "TestNode3", "TestNode4", "TestNode5", "TestNode6" }, Workflow = workflowName });
tasks.Add(new Task { Type = "testTaskType2", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "testTaskType", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "testTaskType2", Name = "TestNode5", Payload = "Node5", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "testTaskType", Name = "TestNode6", Payload = "Node6", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
var workflow = new Workflow { Name = workflowName, Tasks = tasks };
wm.PushWorkflow(workflow);
var workflowWasCompleted = complete.WaitOne(2000);
Assert.IsTrue(workflowWasCompleted);
Console.WriteLine("WM events"); foreach (var ev in events) Console.WriteLine("Event: " + ev);
Assert.AreEqual(6, th.TaskRunCount);
}
}
public void RunsTasksInPriorityOrder()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new TestTaskHandler();
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
{
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(new TaskName("TestNode1"), new Payload("Node1"), NoType, new TaskPriority(3), EmptyTaskList, EmptyTaskList);
workflow.AddTask(new TaskName("TestNode2"), new Payload("Node2"), NoType, new TaskPriority(2), EmptyTaskList, EmptyTaskList);
workflow.AddTask(new TaskName("TestNode3"), new Payload("Node3"), NoType, new TaskPriority(1), EmptyTaskList, EmptyTaskList);
workflow.AddTask(new TaskName("TestNode4"), new Payload("Node4"), NoType, new TaskPriority(4), EmptyTaskList, EmptyTaskList);
workflow.AddTask(new TaskName("TestNode5"), new Payload("Node5"), NoType, new TaskPriority(6), EmptyTaskList, EmptyTaskList);
workflow.AddTask(new TaskName("TestNode6"), new Payload("Node6"), NoType, new TaskPriority(5), EmptyTaskList, EmptyTaskList);
wm.PushWorkflow(workflow);
var workflowWasCompleted = complete.WaitOne(2000);
Assert.IsTrue(workflowWasCompleted);
Console.WriteLine("WM events"); foreach (var ev in events) Console.WriteLine("Event: " + ev);
Assert.AreEqual(6, th.TaskRunCount);
Assert.AreEqual("Node3", th.Payloads[0]);
Assert.AreEqual("Node2", th.Payloads[1]);
Assert.AreEqual("Node1", th.Payloads[2]);
Assert.AreEqual("Node4", th.Payloads[3]);
Assert.AreEqual("Node6", th.Payloads[4]);
Assert.AreEqual("Node5", th.Payloads[5]);
}
}
public void CanCleanUpAfterAWorkflow()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanCleanUpAfterAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new TestTaskHandler();
var complete = new ManualResetEvent(false);
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { complete.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
{
var workflowName = "TestWorkflow";
var tasks = new List<Task>();
tasks.Add(new Task { Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode2", Payload = "Node2", Parents = new string[] { "TestNode1" }, Children = new string[] { "TestNode3", "TestNode4", "TestNode5", "TestNode6" }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode5", Payload = "Node5", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode6", Payload = "Node6", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
var workflow = new Workflow { Name = workflowName, Tasks = tasks };
var workflowId = wm.PushWorkflow(workflow);
var workflowCompleted = complete.WaitOne(2000); // machine-performance dependent, but 2 seconds is a long time
Assert.IsTrue(workflowCompleted);
var info = wm.FetchWorkflowInformation(workflowId.ToString());
Assert.AreEqual("1", info.Id);
Assert.AreEqual(6, info.Tasks.Count);
wm.CleanUp(workflowId.ToString());
db = _mux.GetDatabase();
for (var t = 0; t < 6; t++)
{
Assert.IsFalse(db.KeyExists("task:" + t));
Assert.IsFalse(db.KeyExists("parents-" + t));
Assert.IsFalse(db.KeyExists("children-" + t));
Assert.IsFalse(db.SetContains("tasks", t));
Assert.IsFalse(db.SetContains("submitted", t));
Assert.IsFalse(db.SetContains("complete", t));
Assert.IsFalse(db.SetContains("failed", t));
Assert.IsFalse(db.SetContains("abandoned", t));
Assert.AreEqual(0, db.ListRemove("running", t));
}
Assert.AreEqual(0, db.ListRemove("workflowComplete", "1"));
Assert.AreEqual(0, db.ListRemove("workflowFailed", "1"));
Assert.IsFalse(db.KeyExists("workflow-tasks-1"));
Assert.IsFalse(db.KeyExists("workflow-remaining-1"));
Assert.IsFalse(db.SetContains("workflows", "1"));
Assert.IsFalse(db.KeyExists("submitted:1"));
Assert.IsFalse(db.KeyExists("running:1"));
}
}
public void CanCleanUpAfterAWorkflow()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanCleanUpAfterAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new TestTaskHandler();
var complete = new ManualResetEvent(false);
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { complete.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
{
var t1 = new TaskName("TestNode1");
var t2 = new TaskName("TestNode2");
var t3 = new TaskName("TestNode3");
var t4 = new TaskName("TestNode4");
var t5 = new TaskName("TestNode5");
var t6 = new TaskName("TestNode6");
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
var workflowId = wm.PushWorkflow(workflow);
var workflowCompleted = complete.WaitOne(2000); // machine-performance dependent, but 2 seconds is a long time
Assert.IsTrue(workflowCompleted);
var info = wm.FetchWorkflowInformation(workflowId.ToString());
Assert.AreEqual("1", info.Id);
Assert.AreEqual(6, info.Tasks.Count);
wm.CleanUp(workflowId.ToString());
db = _mux.GetDatabase();
for (var t = 0; t < 6; t++)
{
Assert.IsFalse(db.KeyExists("task:" + t));
Assert.IsFalse(db.KeyExists("parents-" + t));
Assert.IsFalse(db.KeyExists("children-" + t));
Assert.IsFalse(db.SetContains("tasks", t));
Assert.IsFalse(db.SetContains("submitted", t));
Assert.IsFalse(db.SetContains("complete", t));
Assert.IsFalse(db.SetContains("failed", t));
Assert.IsFalse(db.SetContains("abandoned", t));
Assert.AreEqual(0, db.ListRemove("running", t));
}
Assert.AreEqual(0, db.ListRemove("workflowComplete", "1"));
Assert.AreEqual(0, db.ListRemove("workflowFailed", "1"));
Assert.IsFalse(db.KeyExists("workflow-tasks-1"));
Assert.IsFalse(db.KeyExists("workflow-remaining-1"));
Assert.IsFalse(db.SetContains("workflows", "1"));
Assert.IsFalse(db.KeyExists("submitted:1"));
Assert.IsFalse(db.KeyExists("running:1"));
}
}
public void IfATaskIsMarkedFailed_WorfklowFailsAndNoMoreTasksSubmitted()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"IfATaskIsMarkedFailed_WorfklowFailsAndNoMoreTasksSubmitted\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new ErroringTestTaskHandler(2);
var complete = new ManualResetEvent(false);
var failed = new ManualResetEvent(false);
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { complete.Set(); };
wh.WorkflowFailed += (s, w) => { failed.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
{
var workflowName = "TestWorkflow";
var tasks = new List<Task>();
tasks.Add(new Task { Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode2", Payload = "Node2", Parents = new string[] { "TestNode1" }, Children = new string[] { "TestNode3", "TestNode4" }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
var workflow = new Workflow { Name = workflowName, Tasks = tasks };
wm.PushWorkflow(workflow);
var result = WaitHandle.WaitAny(new[] { failed, complete }, 2000);
Assert.AreEqual(0, result);
Assert.AreEqual(2, th.TaskRunCount); // only 2 get sent through the task handler; second fails, so no children are executed
}
}
public void CanSubmitAWorkflowAsync()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new BlockingTaskHandler();
th.LetRun.Set();
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
{
var t1 = new TaskName("TestNode1");
var t2 = new TaskName("TestNode2");
var t3 = new TaskName("TestNode3");
var t4 = new TaskName("TestNode4");
var t5 = new TaskName("TestNode5");
var t6 = new TaskName("TestNode6");
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
var id = wm.PushWorkflowAsync(workflow);
id.Wait();
}
db.ScriptEvaluate("redis.call(\"flushdb\")");
}
public void CanSubmitAWorkflowAsync()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new BlockingTaskHandler();
th.LetRun.Set();
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
{
var workflowName = "TestWorkflow";
var tasks = new List<Task>();
tasks.Add(new Task { Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode2", Payload = "Node2", Parents = new string[] { "TestNode1" }, Children = new string[] { "TestNode3", "TestNode4", "TestNode5", "TestNode6" }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode3", Payload = "Node3", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode4", Payload = "Node4", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode5", Payload = "Node5", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
tasks.Add(new Task { Type = "", Name = "TestNode6", Payload = "Node6", Parents = new string[] { "TestNode2" }, Children = new string[] { }, Workflow = workflowName });
var workflow = new Workflow { Name = workflowName, Tasks = tasks };
var id = wm.PushWorkflowAsync(workflow);
id.Wait();
}
db.ScriptEvaluate("redis.call(\"flushdb\")");
}
public void CanResetOwnTasks()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new BlockingTaskHandler("1");
var th2 = new BlockingTaskHandler("2");
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
var t1 = new TaskName("TestNode1");
var t2 = new TaskName("TestNode2");
var t3 = new TaskName("TestNode3");
var t4 = new TaskName("TestNode4");
var t5 = new TaskName("TestNode5");
var t6 = new TaskName("TestNode6");
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new[] { t1 }, new[] { t3, t4, t5, t6 });
workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
{
wm.PushWorkflow(workflow);
// we've picked up the first task, and don't want to pick up any more
th.Gate.WaitOne();
Assert.AreEqual(0, db.ListLength("submitted"));
}
// create a new wm to simulate a restart of a component
using (var wm2 = new WorkflowManagement(_mux, th2, wh, new WorkflowManagementId("test"), null, new Lua()))
{
Assert.AreEqual(1, db.SortedSetLength("submitted"));
th.Abort.Set();
th2.LetRun.Set();
complete.WaitOne();
}
db.ScriptEvaluate("redis.call(\"flushdb\")");
}
public void CanQueryForOwnTasks()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanSubmitAndRunAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new BlockingTaskHandler();
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, "test", null, new Lua()))
{
var workflowName = "TestWorkflow";
var tasks = new List<Task>();
tasks.Add(new Task { Type = "", Name = "TestNode1", Payload = "Node1", Parents = new string[] { }, Children = new string[] { "TestNode2" }, Workflow = workflowName });
var workflow = new Workflow { Name = workflowName, Tasks = tasks };
wm.PushWorkflow(workflow);
var taskWasStarted = th.Gate.WaitOne(2000);
Assert.IsTrue(taskWasStarted);
var myTasks = wm.FindTasks();
Assert.AreEqual(1, myTasks.Length);
Assert.AreEqual("1", myTasks[0]);
}
db.ScriptEvaluate("redis.call(\"flushdb\")");
}
public void CanPauseAWorkflow()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanPauseAWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new BlockingTaskHandler("1");
var complete = new ManualResetEvent(false);
var events = new List<string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
var t1 = new TaskName("TestNode1");
var t2 = new TaskName("TestNode2");
var t3 = new TaskName("TestNode3");
var t4 = new TaskName("TestNode4");
var t5 = new TaskName("TestNode5");
var t6 = new TaskName("TestNode6");
var type1 = new TaskType("testTaskType");
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new[] { t2 });
workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, EmptyTaskList, new[] { t3, t4, t5, t6 });
workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t5, new Payload("Node5"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
workflow.AddTask(t6, new Payload("Node6"), NoType, SimplePriority, new[] { t2 }, EmptyTaskList);
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), null, new Lua()))
{
var workflowId = wm.PushWorkflow(workflow);
var waitResult = th.Gate.WaitOne(/*2000*/);
Assert.IsTrue(waitResult);
Assert.AreEqual(1, db.SortedSetLength("submitted"));
Assert.AreEqual(1, db.SetLength("running"));
wm.PauseWorkflow(workflowId);
var expected = new List<string> { "1", "2" };
var state = "paused";
CheckSetContent(db, expected, state);
// The running task will run bang into the new state
th.LetRun.Set();
Thread.Sleep(500); // gah, honestly. I won't get a signal because no new tasks should be submitted
expected = new List<string> { "2" };
state = "paused";
CheckSetContent(db, expected, state);
expected = new List<string> { "2" };
state = "running";
CheckSetContent(db, expected, state, false);
// If this is checked before the task has completed then there'll be two paused tasks
// We've got no hook into the completion event as yet
Assert.AreEqual(1, db.SetLength("paused"));
Assert.AreEqual(1, db.SetLength("paused:1"));
Assert.AreEqual(0, db.SortedSetLength("submitted"));
Assert.AreEqual(0, db.SetLength("running"));
}
db.ScriptEvaluate("redis.call(\"flushdb\")");
}
public void CanCleanUpAfterAFailedWorkflow()
{
var db = _mux.GetDatabase();
db.ScriptEvaluate("print(\"CanCleanUpAfterAFailedWorkflow\")");
db.ScriptEvaluate("redis.call(\"flushdb\")");
var th = new ErroringTestWithLongRunnerTaskHandler("Node4", "Node3");
var complete = new ManualResetEvent(false);
var failed = new ManualResetEvent(false);
var exception = new ManualResetEvent(false);
var events = new List <string>();
var wh = new WorkflowHandler();
wh.WorkflowComplete += (s, w) => { events.Add("complete"); complete.Set(); };
wh.WorkflowFailed += (s, w) => { events.Add("failed"); failed.Set(); };
EventHandler <Exception> eh = (s, e) => { events.Add(e.Message); exception.Set(); };
using (var wm = new WorkflowManagement(_mux, th, wh, new WorkflowManagementId("test"), eh))
{
var t1 = new TaskName("TestNode1");
var t2 = new TaskName("TestNode2");
var t3 = new TaskName("TestNode3");
var t4 = new TaskName("TestNode4");
var workflow = new Workflow(new WorkflowName("TestWorkflow"));
workflow.AddTask(t1, new Payload("Node1"), NoType, SimplePriority, EmptyTaskList, new [] { t2 });
workflow.AddTask(t2, new Payload("Node2"), NoType, SimplePriority, new [] { t1 }, new [] { t3, t4 });
workflow.AddTask(t3, new Payload("Node3"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
workflow.AddTask(t4, new Payload("Node4"), NoType, SimplePriority, new [] { t2 }, EmptyTaskList);
var workflowId = wm.PushWorkflow(workflow);
var result = WaitHandle.WaitAny(new[] { exception, failed, complete }, 2000);
if (result == 0)
{
throw new Exception("Failed in workflow operation"); // in a real app, handler could add exception to a list to process
}
// In principle this is a failed workflow, as one of the tasks will fail
// However, if all tasks get started, then the succeeding leaf node finishes first, this will be marked successful
// Is a workflow considered complete when all tasks have started, or when all tasks have finished?
// CHoose latter.
Console.WriteLine("TaskHandler events"); foreach (var ev in th.Events)
{
Console.WriteLine("TH event: " + ev);
}
Console.WriteLine("WM events"); foreach (var ev in events)
{
Console.WriteLine("Event: " + ev);
}
Assert.AreEqual(1, result);
wm.CleanUp(workflowId.ToString());
}
// Should leave highwatermark id keys alone
db = _mux.GetDatabase();
for (var t = 0; t < 4; t++)
{
Assert.IsFalse(db.KeyExists("task:" + t));
Assert.IsFalse(db.KeyExists("parents-" + t));
Assert.IsFalse(db.KeyExists("children-" + t));
Assert.IsFalse(db.SetContains("tasks", t));
Assert.IsFalse(db.SetContains("submitted", t));
Assert.IsFalse(db.SetContains("complete", t));
Assert.IsFalse(db.SetContains("failed", t));
if (t == 3)
{
Assert.IsTrue(db.SetContains("abandoned", t));
db.SetRemove("abandoned", t);
}
else
{
Assert.IsFalse(db.SetContains("abandoned", t));
}
Assert.AreEqual(false, db.SetRemove("running", t));
}
Assert.AreEqual(0, db.ListRemove("workflowComplete", "1"));
Assert.AreEqual(0, db.ListRemove("workflowFailed", "1"));
Assert.IsFalse(db.KeyExists("workflow-tasks-1"));
Assert.IsFalse(db.KeyExists("workflow-remaining-1"));
Assert.IsFalse(db.SetContains("workflows", "1"));
}
