public void LiveExecutionStatus_Insert_Null()
{
LiveExecutionStatus les = new LiveExecutionStatus()
{
Inserted = DateTime.Now,
LastUpdate = DateTime.Now,
ScheduleID = null,
ServerID = 1,
TaskID = 1
};
using (SqlConnection conn = new SqlConnection(Config.CONNECTION_STRING))
{
conn.Open();
using (var trans = conn.BeginTransaction())
{
LiveExecutionStatus.Insert(conn, trans, les);
trans.Commit();
}
}
}
protected void DequeueTasksThread()
{
while (true)
{
try
{
log.TraceFormat("{0:S} - Check for tasks to start", this.ToString());
List <LiveExecutionStatus> lLessStarted = new List <LiveExecutionStatus>();
// Here we get the first task to start, and start it.
// NewTask could raise an exception if the task constructon fails
// to parse the payload. We should avoid letting this starve the queue and put it in failed state.
// In order to do so, we commit the transaction before instantiating the Task's watchdog.
using (SqlConnection conn = new SqlConnection(ConnectionString))
{
conn.Open();
// Start getting a peek on running tasks. We do not care of phantom reads so we
// just settle for ReadCommitted isolation level.
using (SqlTransaction trans = conn.BeginTransaction(System.Data.IsolationLevel.ReadCommitted))
{
// now we lock the ExecutionQueueItem exclusively.
List <ExecutionQueueItem> lEqis = ExecutionQueueItem.GetAll <ExecutionQueueItem>(conn, trans, LockType.TableLockX);
List <LiveExecutionStatus> lLess = LiveExecutionStatus.GetAll <LiveExecutionStatus>(conn, trans, LockType.Default);
// now for each task in the queue find if it can be started and the start it.
foreach (var eqi in lEqis)
{
#region Task to start detection with concurrency limits
var task = Database.Task.GetByID <Task>(conn, trans, eqi.TaskID);
var lGlobalExecutions = lLess.FindAll(x => x.TaskID == eqi.TaskID);
var lLocalExecutions = lGlobalExecutions.FindAll(x => x.ServerID == this.ID);
// we need to count the executions we have started in this transaction as they will not
// show in the previous count.
var lJustStartedExecutions = lLessStarted.FindAll(x => x.TaskID == eqi.TaskID);
int iGlobalExecutionsCount = lGlobalExecutions.Count;
int iLocalExecutionsCount = lLocalExecutions.Count;
var iJustStartedExecutions = lJustStartedExecutions.Count;
if (
(task.ConcurrencyLimitGlobal == 0 || task.ConcurrencyLimitGlobal > iGlobalExecutionsCount) &&
(task.ConcurrencyLimitSameInstance == 0 || task.ConcurrencyLimitSameInstance > (iLocalExecutionsCount + iJustStartedExecutions)) // do not forget just started entries!
)
{
log.InfoFormat("Preparing to start task {0:S}", task.ToString());
// add to live table
var les = new LiveExecutionStatus(eqi.TaskID, this.ID, eqi.ScheduleID);
LiveExecutionStatus.Insert(conn, trans, les);
// remove from pending execution queue
if (!ExecutionQueueItem.Delete(conn, trans, eqi))
{
throw new Exceptions.DatabaseConcurrencyException <ExecutionQueueItem>("Delete", eqi);
}
lLessStarted.Add(les);
}
else
{
log.DebugFormat("Task not started: current situation ({0:N0} / {1:N0}), task {2:S}", iGlobalExecutionsCount, iLocalExecutionsCount + iJustStartedExecutions, task.ToString());
}
#endregion
}
trans.Commit();
}
}
// start the execution. As this call might fail we handle the failue in place.
// If failed the task is considered deququed succesfully. It's up to the
// workflow manager to handle this kind of failures.
foreach (var les in lLessStarted)
{
Task task;
try
{
#region Task thread execution start
using (SqlConnection conn = new SqlConnection(ConnectionString))
{
conn.Open();
task = Database.Task.GetByID <Task>(conn, null, les.TaskID);
}
// get the task for the configuration payload
log.DebugFormat("Starting enqueued task {0:S} for LES {1:S}", task.ToString(), les.ToString());
string detemplatedPayload = DetemplatePayload(task.Payload);
var wd = ExecutionTasks.Factory.NewTask(this, task.Type, detemplatedPayload, les);
wd.Start();
log.InfoFormat("Started task {0:S} as live execution status {1:S}", task.ToString(), les.ToString());
#endregion
}
catch (Exception exce)
{
log.WarnFormat("LES {0:S} failed at startup: {0:S}", les.ToString(), exce.ToString());
#region Set as failed during startup
using (SqlConnection conn = new SqlConnection(this.ConnectionString))
{
conn.Open();
using (var trans = conn.BeginTransaction())
{
DeadExecutionStatus des = new DeadExecutionStatus(les, TaskStatus.ExceptionAtStartup, exce.ToString());
DeadExecutionStatus.Insert(conn, trans, des);
if (!ExecutionQueueItem.Delete(conn, trans, les))
{
throw new Exceptions.DatabaseConcurrencyException <LiveExecutionStatus>("Delete", les);
}
trans.Commit();
}
}
#endregion
}
}
}
catch (Exception exce) // we must catch unhandled exceptions to keep this thread alive
{
log.ErrorFormat("Unhandled exception during DequeueTasksThread: {0:S}", exce.ToString());
}
Thread.Sleep(int.Parse(Configuration["SERVER_POLL_TASK_QUEUE_SLEEP_MS"]));
}
}
public static Watchdog NewTask(Server server, string taskType, string configPayload, LiveExecutionStatus les)
{
log.DebugFormat("Factory.NewTask({0:S}, {1:S}) called", server.ToString(), les.ToString());
var typeToCreate = Tasks.FirstOrDefault(t => t.Name == taskType);
if (typeToCreate == null)
{
throw new Exceptions.UnsupportedTaskException(taskType);
}
GenericTask task = (GenericTask)Activator.CreateInstance(typeToCreate);
task.ParseConfiguration(configPayload);
return(new Watchdog(server, task, les));
}
protected void DeadTasksThread()
{
while (true)
{
try
{
log.TraceFormat("{0:S} - Check for dead tasks", this.ToString());
// a task is dead if there is no update in the xxx milliseconds (1 minute default)
int iTimeoutMilliseconds = int.Parse(Configuration["TASK_MAXIMUM_UPDATE_LAG_MS"]);
using (SqlConnection conn = new SqlConnection(ConnectionString))
{
conn.Open();
using (SqlTransaction trans = conn.BeginTransaction(System.Data.IsolationLevel.ReadCommitted))
{
var lExpired = LiveExecutionStatus.GetAndLockExpired(conn, trans, iTimeoutMilliseconds);
foreach (var les in lExpired)
{
#region Mark execution as expired
// insert into dead table
DeadExecutionStatus des = new DeadExecutionStatus(les, TaskStatus.Dead, null);
DeadExecutionStatus.Insert(conn, trans, des);
log.WarnFormat("Setting LiveExecutionStatus {0:S} as dead", les.ToString());
// remove from live table
if (!LiveExecutionStatus.Delete(conn, trans, les))
{
throw new Exceptions.DatabaseConcurrencyException <LiveExecutionStatus>("Delete", les);
}
// if required, reenqueue
var task = YoctoScheduler.Core.Database.Task.GetByID <Task>(conn, trans, les.TaskID);
if (task.ReenqueueOnDead)
{
log.InfoFormat("Reenqueuing {0:S} as requested", task.ToString());
ExecutionQueueItem eqi = new ExecutionQueueItem()
{
TaskID = task.ID,
Priority = Priority.Normal,
ScheduleID = les.ScheduleID,
Inserted = DateTime.Now
};
ExecutionQueueItem.Insert(conn, trans, eqi);
}
#endregion
}
trans.Commit();
}
}
Thread.Sleep(int.Parse(Configuration["SERVER_POLL_DISABLE_DEAD_TASKS_SLEEP_MS"]));
}
catch (Exception exce) // catch all to keep the thread alive
{
log.ErrorFormat("Unhandled exception during DeadTasksThread: {0:S}", exce.ToString());
}
}
}
public Watchdog(Server Server, GenericTask task, LiveExecutionStatus LiveExecutionStatus)
{
this.Server = Server;
this.Task = task;
this.LiveExecutionStatus = LiveExecutionStatus;
}
