protected void TasksScheduledThread() { while (true) { log.TraceFormat("{0:S} - Check for tasks to schedule", this.ToString()); try { using (SqlConnection conn = new SqlConnection(ConnectionString)) { conn.Open(); using (SqlTransaction trans = conn.BeginTransaction(System.Data.IsolationLevel.Serializable)) { // Get enabled schedules var lSchedules = Schedule.GetAndLockEnabledNotRunning(conn, trans); #region look for schedules to fire lSchedules.ForEach(sched => { NCrontab.CrontabSchedule cs = NCrontab.CrontabSchedule.Parse(sched.Cron); if (cs.GetNextOccurrence(LastScheduleCheck) < DateTime.Now && (sched.LastFired < cs.GetNextOccurrence(LastScheduleCheck))) { var task = YoctoScheduler.Core.Database.Task.GetByID <Task>(conn, trans, sched.TaskID); log.InfoFormat("Starting schedulation {0:S} due to cron {1:S}", task.ToString(), sched.ToString()); // save schedule fire time sched.LastFired = DateTime.Now; Schedule.Update(conn, trans, sched); ExecutionQueueItem eqi = new ExecutionQueueItem() { TaskID = task.ID, Priority = Priority.Normal, ScheduleID = sched.ID, Inserted = DateTime.Now }; ExecutionQueueItem.Insert(conn, trans, eqi); log.InfoFormat("Execution enqueued {0:S}", eqi.ToString()); } }); #endregion trans.Commit(); } } LastScheduleCheck = DateTime.Now; Thread.Sleep(int.Parse(Configuration["SERVER_POLL_TASK_SCHEDULER_SLEEP_MS"])); } catch (Exception exce) // catch all to keep the thread alive { log.ErrorFormat("Unhandled exception during TasksScheduledThread: {0:S}", exce.ToString()); } } }
public void ExecutionQueueItem_InsertNull() { ExecutionQueueItem eqi = new ExecutionQueueItem() { Inserted = DateTime.Now, Priority = Core.Priority.Lowest, TaskID = 1 }; using (SqlConnection conn = new SqlConnection(Config.CONNECTION_STRING)) { conn.Open(); using (var trans = conn.BeginTransaction()) { ExecutionQueueItem.Insert(conn, trans, eqi); trans.Commit(); } } }
protected void DequeueTasksThread() { while (true) { try { log.TraceFormat("{0:S} - Check for tasks to start", this.ToString()); List <LiveExecutionStatus> lLessStarted = new List <LiveExecutionStatus>(); // Here we get the first task to start, and start it. // NewTask could raise an exception if the task constructon fails // to parse the payload. We should avoid letting this starve the queue and put it in failed state. // In order to do so, we commit the transaction before instantiating the Task's watchdog. using (SqlConnection conn = new SqlConnection(ConnectionString)) { conn.Open(); // Start getting a peek on running tasks. We do not care of phantom reads so we // just settle for ReadCommitted isolation level. using (SqlTransaction trans = conn.BeginTransaction(System.Data.IsolationLevel.ReadCommitted)) { // now we lock the ExecutionQueueItem exclusively. List <ExecutionQueueItem> lEqis = ExecutionQueueItem.GetAll <ExecutionQueueItem>(conn, trans, LockType.TableLockX); List <LiveExecutionStatus> lLess = LiveExecutionStatus.GetAll <LiveExecutionStatus>(conn, trans, LockType.Default); // now for each task in the queue find if it can be started and the start it. foreach (var eqi in lEqis) { #region Task to start detection with concurrency limits var task = Database.Task.GetByID <Task>(conn, trans, eqi.TaskID); var lGlobalExecutions = lLess.FindAll(x => x.TaskID == eqi.TaskID); var lLocalExecutions = lGlobalExecutions.FindAll(x => x.ServerID == this.ID); // we need to count the executions we have started in this transaction as they will not // show in the previous count. var lJustStartedExecutions = lLessStarted.FindAll(x => x.TaskID == eqi.TaskID); int iGlobalExecutionsCount = lGlobalExecutions.Count; int iLocalExecutionsCount = lLocalExecutions.Count; var iJustStartedExecutions = lJustStartedExecutions.Count; if ( (task.ConcurrencyLimitGlobal == 0 || task.ConcurrencyLimitGlobal > iGlobalExecutionsCount) && (task.ConcurrencyLimitSameInstance == 0 || task.ConcurrencyLimitSameInstance > (iLocalExecutionsCount + iJustStartedExecutions)) // do not forget just started entries! ) { log.InfoFormat("Preparing to start task {0:S}", task.ToString()); // add to live table var les = new LiveExecutionStatus(eqi.TaskID, this.ID, eqi.ScheduleID); LiveExecutionStatus.Insert(conn, trans, les); // remove from pending execution queue if (!ExecutionQueueItem.Delete(conn, trans, eqi)) { throw new Exceptions.DatabaseConcurrencyException <ExecutionQueueItem>("Delete", eqi); } lLessStarted.Add(les); } else { log.DebugFormat("Task not started: current situation ({0:N0} / {1:N0}), task {2:S}", iGlobalExecutionsCount, iLocalExecutionsCount + iJustStartedExecutions, task.ToString()); } #endregion } trans.Commit(); } } // start the execution. As this call might fail we handle the failue in place. // If failed the task is considered deququed succesfully. It's up to the // workflow manager to handle this kind of failures. foreach (var les in lLessStarted) { Task task; try { #region Task thread execution start using (SqlConnection conn = new SqlConnection(ConnectionString)) { conn.Open(); task = Database.Task.GetByID <Task>(conn, null, les.TaskID); } // get the task for the configuration payload log.DebugFormat("Starting enqueued task {0:S} for LES {1:S}", task.ToString(), les.ToString()); string detemplatedPayload = DetemplatePayload(task.Payload); var wd = ExecutionTasks.Factory.NewTask(this, task.Type, detemplatedPayload, les); wd.Start(); log.InfoFormat("Started task {0:S} as live execution status {1:S}", task.ToString(), les.ToString()); #endregion } catch (Exception exce) { log.WarnFormat("LES {0:S} failed at startup: {0:S}", les.ToString(), exce.ToString()); #region Set as failed during startup using (SqlConnection conn = new SqlConnection(this.ConnectionString)) { conn.Open(); using (var trans = conn.BeginTransaction()) { DeadExecutionStatus des = new DeadExecutionStatus(les, TaskStatus.ExceptionAtStartup, exce.ToString()); DeadExecutionStatus.Insert(conn, trans, des); if (!ExecutionQueueItem.Delete(conn, trans, les)) { throw new Exceptions.DatabaseConcurrencyException <LiveExecutionStatus>("Delete", les); } trans.Commit(); } } #endregion } } } catch (Exception exce) // we must catch unhandled exceptions to keep this thread alive { log.ErrorFormat("Unhandled exception during DequeueTasksThread: {0:S}", exce.ToString()); } Thread.Sleep(int.Parse(Configuration["SERVER_POLL_TASK_QUEUE_SLEEP_MS"])); } }
protected void DeadTasksThread() { while (true) { try { log.TraceFormat("{0:S} - Check for dead tasks", this.ToString()); // a task is dead if there is no update in the xxx milliseconds (1 minute default) int iTimeoutMilliseconds = int.Parse(Configuration["TASK_MAXIMUM_UPDATE_LAG_MS"]); using (SqlConnection conn = new SqlConnection(ConnectionString)) { conn.Open(); using (SqlTransaction trans = conn.BeginTransaction(System.Data.IsolationLevel.ReadCommitted)) { var lExpired = LiveExecutionStatus.GetAndLockExpired(conn, trans, iTimeoutMilliseconds); foreach (var les in lExpired) { #region Mark execution as expired // insert into dead table DeadExecutionStatus des = new DeadExecutionStatus(les, TaskStatus.Dead, null); DeadExecutionStatus.Insert(conn, trans, des); log.WarnFormat("Setting LiveExecutionStatus {0:S} as dead", les.ToString()); // remove from live table if (!LiveExecutionStatus.Delete(conn, trans, les)) { throw new Exceptions.DatabaseConcurrencyException <LiveExecutionStatus>("Delete", les); } // if required, reenqueue var task = YoctoScheduler.Core.Database.Task.GetByID <Task>(conn, trans, les.TaskID); if (task.ReenqueueOnDead) { log.InfoFormat("Reenqueuing {0:S} as requested", task.ToString()); ExecutionQueueItem eqi = new ExecutionQueueItem() { TaskID = task.ID, Priority = Priority.Normal, ScheduleID = les.ScheduleID, Inserted = DateTime.Now }; ExecutionQueueItem.Insert(conn, trans, eqi); } #endregion } trans.Commit(); } } Thread.Sleep(int.Parse(Configuration["SERVER_POLL_DISABLE_DEAD_TASKS_SLEEP_MS"])); } catch (Exception exce) // catch all to keep the thread alive { log.ErrorFormat("Unhandled exception during DeadTasksThread: {0:S}", exce.ToString()); } } }
public void Enqueue(ExecutionQueueItem item) { _queue.Add(item, _cancellationToken); }