/// <summary>
/// Initializes the job execution context with given scheduler and bundle.
/// </summary>
/// <param name="sched">The scheduler.</param>
/// <param name="firedBundle">The bundle offired triggers.</param>
public virtual void Initialize(QuartzScheduler sched, TriggerFiredBundle firedBundle)
{
qs = sched;
IJob job;
JobDetail jobDetail = firedBundle.JobDetail;
try
{
job = sched.JobFactory.NewJob(firedBundle);
}
catch (SchedulerException se)
{
sched.NotifySchedulerListenersError(string.Format(CultureInfo.InvariantCulture, "An error occured instantiating job to be executed. job= '{0}'", jobDetail.FullName), se);
throw;
}
catch (Exception e)
{
SchedulerException se = new SchedulerException(string.Format(CultureInfo.InvariantCulture, "Problem instantiating type '{0}'", jobDetail.JobType.FullName), e);
sched.NotifySchedulerListenersError(string.Format(CultureInfo.InvariantCulture, "An error occured instantiating job to be executed. job= '{0}'", jobDetail.FullName), se);
throw se;
}
jec = new JobExecutionContext(scheduler, firedBundle, job);
}
/// <summary>
/// Initializes the job execution context with given scheduler and bundle.
/// </summary>
/// <param name="sched">The scheduler.</param>
/// <param name="cancellationToken">The cancellation instruction.</param>
public virtual async Task Initialize(
QuartzScheduler sched,
CancellationToken cancellationToken = default)
{
qs = sched;
IJob job;
IJobDetail jobDetail = firedTriggerBundle.JobDetail;
try
{
job = sched.JobFactory.NewJob(firedTriggerBundle, scheduler);
}
catch (SchedulerException se)
{
await sched.NotifySchedulerListenersError($"An error occurred instantiating job to be executed. job= '{jobDetail.Key}'", se, cancellationToken).ConfigureAwait(false);
throw;
}
catch (Exception e)
{
SchedulerException se = new SchedulerException($"Problem instantiating type '{jobDetail.JobType.FullName}: {e.Message}'", e);
await sched.NotifySchedulerListenersError($"An error occurred instantiating job to be executed. job= '{jobDetail.Key}, message={e.Message}'", se, cancellationToken).ConfigureAwait(false);
throw se;
}
jec = new JobExecutionContextImpl(scheduler, firedTriggerBundle, job);
}
/// <summary>
/// Notifies the scheduler about misfired trigger.
/// </summary>
/// <param name="trigger">The trigger that misfired.</param>
public virtual async Task NotifyTriggerListenersMisfired(ITrigger trigger)
{
try
{
await sched.NotifyTriggerListenersMisfired(trigger).ConfigureAwait(false);
}
catch (SchedulerException se)
{
log.ErrorException("Error notifying listeners of trigger misfire.", se);
await sched.NotifySchedulerListenersError("Error notifying listeners of trigger misfire.", se).ConfigureAwait(false);
}
}
/// <summary>
/// Notifies the scheduler about misfired trigger.
/// </summary>
/// <param name="trigger">The trigger that misfired.</param>
public virtual void NotifyTriggerListenersMisfired(ITrigger trigger)
{
try
{
sched.NotifyTriggerListenersMisfired(trigger);
}
catch (SchedulerException se)
{
log.Error("Error notifying listeners of trigger misfire.{0}", se);
sched.NotifySchedulerListenersError("Error notifying listeners of trigger misfire.", se);
}
}
/// <summary>
/// Notifies the scheduler about misfired trigger.
/// </summary>
/// <param name="trigger">The trigger that misfired.</param>
/// <param name="cancellationToken">The cancellation instruction.</param>
public virtual async Task NotifyTriggerListenersMisfired(
ITrigger trigger,
CancellationToken cancellationToken = default(CancellationToken))
{
try
{
await sched.NotifyTriggerListenersMisfired(trigger, cancellationToken).ConfigureAwait(false);
}
catch (SchedulerException se)
{
log.ErrorException("Error notifying listeners of trigger misfire.", se);
await sched.NotifySchedulerListenersError("Error notifying listeners of trigger misfire.", se, cancellationToken).ConfigureAwait(false);
}
}
/// <summary>
/// The main processing loop of the <see cref="QuartzSchedulerThread" />.
/// </summary>
public override void Run()
{
bool lastAcquireFailed = false;
while (!halted)
{
try
{
// check if we're supposed to pause...
lock (sigLock)
{
while (paused && !halted)
{
try
{
// wait until togglePause(false) is called...
Monitor.Wait(sigLock, 1000);
}
catch (ThreadInterruptedException)
{
}
}
if (halted)
{
break;
}
}
int availThreadCount = qsRsrcs.ThreadPool.BlockForAvailableThreads();
if (availThreadCount > 0) // will always be true, due to semantics of blockForAvailableThreads...
{
IList <IOperableTrigger> triggers = null;
DateTimeOffset now = SystemTime.UtcNow();
ClearSignaledSchedulingChange();
try
{
triggers = qsRsrcs.JobStore.AcquireNextTriggers(
now + idleWaitTime, Math.Min(availThreadCount, qsRsrcs.MaxBatchSize), qsRsrcs.BatchTimeWindow);
lastAcquireFailed = false;
if (log.IsDebugEnabled)
{
log.DebugFormat("Batch acquisition of {0} triggers", (triggers == null ? 0 : triggers.Count));
}
}
catch (JobPersistenceException jpe)
{
if (!lastAcquireFailed)
{
qs.NotifySchedulerListenersError("An error occurred while scanning for the next trigger to fire.", jpe);
}
lastAcquireFailed = true;
continue;
}
catch (Exception e)
{
if (!lastAcquireFailed)
{
Log.Error("quartzSchedulerThreadLoop: RuntimeException " + e.Message, e);
}
lastAcquireFailed = true;
continue;
}
if (triggers != null && triggers.Count > 0)
{
now = SystemTime.UtcNow();
DateTimeOffset triggerTime = triggers[0].GetNextFireTimeUtc().Value;
TimeSpan timeUntilTrigger = triggerTime - now;
while (timeUntilTrigger > TimeSpan.FromMilliseconds(2))
{
if (ReleaseIfScheduleChangedSignificantly(triggers, triggerTime))
{
break;
}
lock (sigLock)
{
if (halted)
{
break;
}
if (!IsCandidateNewTimeEarlierWithinReason(triggerTime, false))
{
try
{
// we could have blocked a long while
// on 'synchronize', so we must recompute
now = SystemTime.UtcNow();
timeUntilTrigger = triggerTime - now;
if (timeUntilTrigger > TimeSpan.Zero)
{
Monitor.Wait(sigLock, timeUntilTrigger);
}
}
catch (ThreadInterruptedException)
{
}
}
}
if (ReleaseIfScheduleChangedSignificantly(triggers, triggerTime))
{
break;
}
now = SystemTime.UtcNow();
timeUntilTrigger = triggerTime - now;
}
// this happens if releaseIfScheduleChangedSignificantly decided to release triggers
if (triggers.Count == 0)
{
continue;
}
// set triggers to 'executing'
IList <TriggerFiredResult> bndles = new List <TriggerFiredResult>();
bool goAhead = true;
lock (sigLock)
{
goAhead = !halted;
}
if (goAhead)
{
try
{
IList <TriggerFiredResult> res = qsRsrcs.JobStore.TriggersFired(triggers);
if (res != null)
{
bndles = res;
}
}
catch (SchedulerException se)
{
qs.NotifySchedulerListenersError("An error occurred while firing triggers '" + triggers + "'", se);
// QTZ-179 : a problem occurred interacting with the triggers from the db
// we release them and loop again
foreach (IOperableTrigger t in triggers)
{
qsRsrcs.JobStore.ReleaseAcquiredTrigger(t);
}
continue;
}
}
for (int i = 0; i < bndles.Count; i++)
{
TriggerFiredResult result = bndles[i];
TriggerFiredBundle bndle = result.TriggerFiredBundle;
Exception exception = result.Exception;
IOperableTrigger trigger = triggers[i];
// TODO SQL exception?
if (exception != null && (exception is DbException || exception.InnerException is DbException))
{
Log.Error("DbException while firing trigger " + trigger, exception);
qsRsrcs.JobStore.ReleaseAcquiredTrigger(trigger);
continue;
}
// it's possible to get 'null' if the triggers was paused,
// blocked, or other similar occurrences that prevent it being
// fired at this time... or if the scheduler was shutdown (halted)
if (bndle == null)
{
qsRsrcs.JobStore.ReleaseAcquiredTrigger(trigger);
continue;
}
// TODO: improvements:
//
// 2- make sure we can get a job runshell before firing trigger, or
// don't let that throw an exception (right now it never does,
// but the signature says it can).
// 3- acquire more triggers at a time (based on num threads available?)
JobRunShell shell = null;
try
{
shell = qsRsrcs.JobRunShellFactory.CreateJobRunShell(bndle);
shell.Initialize(qs);
}
catch (SchedulerException)
{
qsRsrcs.JobStore.TriggeredJobComplete(trigger, bndle.JobDetail, SchedulerInstruction.SetAllJobTriggersError);
continue;
}
if (qsRsrcs.ThreadPool.RunInThread(shell) == false)
{
// this case should never happen, as it is indicative of the
// scheduler being shutdown or a bug in the thread pool or
// a thread pool being used concurrently - which the docs
// say not to do...
Log.Error("ThreadPool.runInThread() return false!");
qsRsrcs.JobStore.TriggeredJobComplete(trigger, bndle.JobDetail, SchedulerInstruction.SetAllJobTriggersError);
}
}
continue; // while (!halted)
}
}
else // if(availThreadCount > 0)
{
// should never happen, if threadPool.blockForAvailableThreads() follows contract
continue;
// while (!halted)
}
DateTimeOffset utcNow = SystemTime.UtcNow();
DateTimeOffset waitTime = utcNow.Add(GetRandomizedIdleWaitTime());
TimeSpan timeUntilContinue = waitTime - utcNow;
lock (sigLock)
{
if (!halted)
{
try
{
// QTZ-336 A job might have been completed in the mean time and we might have
// missed the scheduled changed signal by not waiting for the notify() yet
// Check that before waiting for too long in case this very job needs to be
// scheduled very soon
if (!IsScheduleChanged())
{
Monitor.Wait(sigLock, timeUntilContinue);
}
}
catch (ThreadInterruptedException)
{
}
}
}
}
catch (Exception re)
{
if (Log != null)
{
Log.Error("Runtime error occurred in main trigger firing loop.", re);
}
}
} // while (!halted)
// drop references to scheduler stuff to aid garbage collection...
qs = null;
qsRsrcs = null;
}
/// <summary>
/// The main processing loop of the <see cref="QuartzSchedulerThread" />.
/// </summary>
public override void Run()
{
bool lastAcquireFailed = false;
while (!halted)
{
try
{
// check if we're supposed to pause...
lock (sigLock)
{
while (paused && !halted)
{
try
{
// wait until togglePause(false) is called...
Monitor.Wait(sigLock, 1000);
}
catch (ThreadInterruptedException)
{
}
}
if (halted)
{
break;
}
}
int availTreadCount = qsRsrcs.ThreadPool.BlockForAvailableThreads();
if (availTreadCount > 0) // will always be true, due to semantics of blockForAvailableThreads...
{
Trigger trigger = null;
DateTime now = DateTime.UtcNow;
ClearSignaledSchedulingChange();
try
{
trigger = qsRsrcs.JobStore.AcquireNextTrigger(ctxt, now.Add(idleWaitTime));
lastAcquireFailed = false;
}
catch (JobPersistenceException jpe)
{
if (!lastAcquireFailed)
{
qs.NotifySchedulerListenersError(
"An error occured while scanning for the next trigger to fire.",
jpe);
}
lastAcquireFailed = true;
}
catch (Exception e)
{
if (!lastAcquireFailed)
{
Log.Error("quartzSchedulerThreadLoop: RuntimeException "
+ e.Message, e);
}
lastAcquireFailed = true;
}
if (trigger != null)
{
now = DateTime.UtcNow;
DateTime triggerTime = trigger.GetNextFireTimeUtc().Value;
TimeSpan timeUntilTrigger = triggerTime - now;
while (timeUntilTrigger > TimeSpan.Zero)
{
lock (sigLock)
{
try
{
// we could have blocked a long while
// on 'synchronize', so we must recompute
now = DateTime.UtcNow;
timeUntilTrigger = triggerTime - now;
if (timeUntilTrigger.TotalMilliseconds > 1)
{
Monitor.Wait(sigLock, timeUntilTrigger);
}
}
catch (ThreadInterruptedException)
{
}
}
if (IsScheduleChanged())
{
if (IsCandidateNewTimeEarlierWithinReason(triggerTime))
{
// above call does a clearSignaledSchedulingChange()
try
{
qsRsrcs.JobStore.ReleaseAcquiredTrigger(ctxt, trigger);
}
catch (JobPersistenceException jpe)
{
qs.NotifySchedulerListenersError(
"An error occured while releasing trigger '"
+ trigger.FullName + "'",
jpe);
// db connection must have failed... keep
// retrying until it's up...
ReleaseTriggerRetryLoop(trigger);
}
catch (Exception e)
{
Log.Error(
"releaseTriggerRetryLoop: RuntimeException "
+ e.Message, e);
// db connection must have failed... keep
// retrying until it's up...
ReleaseTriggerRetryLoop(trigger);
}
trigger = null;
break;
}
}
now = DateTime.UtcNow;
timeUntilTrigger = triggerTime - now;
}
if (trigger == null)
{
continue;
}
// set trigger to 'executing'
TriggerFiredBundle bndle = null;
bool goAhead = true;
lock (sigLock)
{
goAhead = !halted;
}
if (goAhead)
{
try
{
bndle = qsRsrcs.JobStore.TriggerFired(ctxt,
trigger);
}
catch (SchedulerException se)
{
qs.NotifySchedulerListenersError(
string.Format(CultureInfo.InvariantCulture,
"An error occured while firing trigger '{0}'",
trigger.FullName), se);
}
catch (Exception e)
{
Log.Error(
string.Format(CultureInfo.InvariantCulture,
"RuntimeException while firing trigger {0}", trigger.FullName),
e);
// db connection must have failed... keep
// retrying until it's up...
ReleaseTriggerRetryLoop(trigger);
}
}
// it's possible to get 'null' if the trigger was paused,
// blocked, or other similar occurrences that prevent it being
// fired at this time... or if the scheduler was shutdown (halted)
if (bndle == null)
{
try
{
qsRsrcs.JobStore.ReleaseAcquiredTrigger(ctxt,
trigger);
}
catch (SchedulerException se)
{
qs.NotifySchedulerListenersError(
string.Format(CultureInfo.InvariantCulture, "An error occured while releasing trigger '{0}'",
trigger.FullName), se);
// db connection must have failed... keep retrying
// until it's up...
ReleaseTriggerRetryLoop(trigger);
}
continue;
}
// TODO: improvements:
//
// 2- make sure we can get a job runshell before firing trigger, or
// don't let that throw an exception (right now it never does,
// but the signature says it can).
// 3- acquire more triggers at a time (based on num threads available?)
JobRunShell shell;
try
{
shell = qsRsrcs.JobRunShellFactory.BorrowJobRunShell();
shell.Initialize(qs, bndle);
}
catch (SchedulerException)
{
try
{
qsRsrcs.JobStore.TriggeredJobComplete(ctxt,
trigger, bndle.JobDetail,
SchedulerInstruction.
SetAllJobTriggersError);
}
catch (SchedulerException se2)
{
qs.NotifySchedulerListenersError(
string.Format(
CultureInfo.InvariantCulture,
"An error occured while placing job's triggers in error state '{0}'",
trigger.FullName), se2);
// db connection must have failed... keep retrying
// until it's up...
ErrorTriggerRetryLoop(bndle);
}
continue;
}
if (qsRsrcs.ThreadPool.RunInThread(shell) == false)
{
try
{
// this case should never happen, as it is indicative of the
// scheduler being shutdown or a bug in the thread pool or
// a thread pool being used concurrently - which the docs
// say not to do...
Log.Error("ThreadPool.runInThread() return false!");
qsRsrcs.JobStore.TriggeredJobComplete(ctxt,
trigger, bndle.JobDetail,
SchedulerInstruction.
SetAllJobTriggersError);
}
catch (SchedulerException se2)
{
qs.NotifySchedulerListenersError(
string.Format(CultureInfo.InvariantCulture,
"An error occured while placing job's triggers in error state '{0}'",
trigger.FullName), se2);
// db connection must have failed... keep retrying
// until it's up...
ReleaseTriggerRetryLoop(trigger);
}
}
continue;
}
}
else
{
// if(availTreadCount > 0)
continue; // should never happen, if threadPool.blockForAvailableThreads() follows contract
}
DateTime utcNow = DateTime.UtcNow;
DateTime waitTime = utcNow.Add(GetRandomizedIdleWaitTime());
TimeSpan timeUntilContinue = waitTime - utcNow;
lock (sigLock)
{
try
{
Monitor.Wait(sigLock, timeUntilContinue);
}
catch (ThreadInterruptedException)
{
}
}
}
catch (Exception re)
{
if (Log != null)
{
Log.Error("Runtime error occured in main trigger firing loop.", re);
}
}
} // loop...
// drop references to scheduler stuff to aid garbage collection...
qs = null;
qsRsrcs = null;
}
/// <summary>
/// The main processing loop of the <see cref="QuartzSchedulerThread" />.
/// </summary>
public async Task Run()
{
int acquiresFailed = 0;
while (!halted)
{
cancellationTokenSource.Token.ThrowIfCancellationRequested();
try
{
// check if we're supposed to pause...
lock (sigLock)
{
while (paused && !halted)
{
try
{
// wait until togglePause(false) is called...
Monitor.Wait(sigLock, 1000);
}
catch (ThreadInterruptedException)
{
}
// reset failure counter when paused, so that we don't
// wait again after unpausing
acquiresFailed = 0;
}
if (halted)
{
break;
}
}
// wait a bit, if reading from job store is consistently
// failing (e.g. DB is down or restarting)..
if (acquiresFailed > 1)
{
try
{
var delay = ComputeDelayForRepeatedErrors(qsRsrcs.JobStore, acquiresFailed);
await Task.Delay(delay);
}
catch
{
}
}
cancellationTokenSource.Token.ThrowIfCancellationRequested();
int availThreadCount = qsRsrcs.ThreadPool.BlockForAvailableThreads();
if (availThreadCount > 0)
{
List <IOperableTrigger> triggers;
DateTimeOffset now = SystemTime.UtcNow();
ClearSignaledSchedulingChange();
try
{
var noLaterThan = now + idleWaitTime;
var maxCount = Math.Min(availThreadCount, qsRsrcs.MaxBatchSize);
triggers = new List <IOperableTrigger>(await qsRsrcs.JobStore.AcquireNextTriggers(noLaterThan, maxCount, qsRsrcs.BatchTimeWindow, CancellationToken.None).ConfigureAwait(false));
acquiresFailed = 0;
if (Log.IsDebugEnabled())
{
Log.DebugFormat("Batch acquisition of {0} triggers", triggers?.Count ?? 0);
}
}
catch (JobPersistenceException jpe)
{
if (acquiresFailed == 0)
{
var msg = "An error occurred while scanning for the next trigger to fire.";
await qs.NotifySchedulerListenersError(msg, jpe, CancellationToken.None).ConfigureAwait(false);
}
if (acquiresFailed < int.MaxValue)
{
acquiresFailed++;
}
continue;
}
catch (Exception e)
{
if (acquiresFailed == 0)
{
Log.ErrorException("quartzSchedulerThreadLoop: RuntimeException " + e.Message, e);
}
if (acquiresFailed < int.MaxValue)
{
acquiresFailed++;
}
continue;
}
if (triggers != null && triggers.Count > 0)
{
now = SystemTime.UtcNow();
DateTimeOffset triggerTime = triggers[0].GetNextFireTimeUtc() !.Value;
TimeSpan timeUntilTrigger = triggerTime - now;
while (timeUntilTrigger > TimeSpan.Zero)
{
if (await ReleaseIfScheduleChangedSignificantly(triggers, triggerTime).ConfigureAwait(false))
{
break;
}
lock (sigLock)
{
if (halted)
{
break;
}
if (!IsCandidateNewTimeEarlierWithinReason(triggerTime, false))
{
try
{
// we could have blocked a long while
// on 'synchronize', so we must recompute
now = SystemTime.UtcNow();
timeUntilTrigger = triggerTime - now;
if (timeUntilTrigger > TimeSpan.Zero)
{
Monitor.Wait(sigLock, timeUntilTrigger);
}
}
catch (ThreadInterruptedException)
{
}
}
}
if (await ReleaseIfScheduleChangedSignificantly(triggers, triggerTime).ConfigureAwait(false))
{
break;
}
now = SystemTime.UtcNow();
timeUntilTrigger = triggerTime - now;
}
// this happens if releaseIfScheduleChangedSignificantly decided to release triggers
if (triggers.Count == 0)
{
continue;
}
// set triggers to 'executing'
List <TriggerFiredResult> bndles = new List <TriggerFiredResult>();
bool goAhead;
lock (sigLock)
{
goAhead = !halted;
}
if (goAhead)
{
try
{
var res = await qsRsrcs.JobStore.TriggersFired(triggers, CancellationToken.None).ConfigureAwait(false);
if (res != null)
{
bndles = res.ToList();
}
}
catch (SchedulerException se)
{
var msg = "An error occurred while firing triggers '" + triggers + "'";
await qs.NotifySchedulerListenersError(msg, se, CancellationToken.None).ConfigureAwait(false);
// QTZ-179 : a problem occurred interacting with the triggers from the db
// we release them and loop again
foreach (IOperableTrigger t in triggers)
{
await qsRsrcs.JobStore.ReleaseAcquiredTrigger(t, CancellationToken.None).ConfigureAwait(false);
}
continue;
}
}
for (int i = 0; i < bndles.Count; i++)
{
TriggerFiredResult result = bndles[i];
var bndle = result.TriggerFiredBundle;
var exception = result.Exception;
IOperableTrigger trigger = triggers[i];
// TODO SQL exception?
if (exception != null && (exception is DbException || exception.InnerException is DbException))
{
Log.ErrorException("DbException while firing trigger " + trigger, exception);
await qsRsrcs.JobStore.ReleaseAcquiredTrigger(trigger, CancellationToken.None).ConfigureAwait(false);
continue;
}
// it's possible to get 'null' if the triggers was paused,
// blocked, or other similar occurrences that prevent it being
// fired at this time... or if the scheduler was shutdown (halted)
if (bndle == null)
{
await qsRsrcs.JobStore.ReleaseAcquiredTrigger(trigger, CancellationToken.None).ConfigureAwait(false);
continue;
}
// TODO: improvements:
//
// 2- make sure we can get a job runshell before firing trigger, or
// don't let that throw an exception (right now it never does,
// but the signature says it can).
// 3- acquire more triggers at a time (based on num threads available?)
JobRunShell shell;
try
{
shell = qsRsrcs.JobRunShellFactory.CreateJobRunShell(bndle);
await shell.Initialize(qs, CancellationToken.None).ConfigureAwait(false);
}
catch (SchedulerException)
{
await qsRsrcs.JobStore.TriggeredJobComplete(trigger, bndle.JobDetail, SchedulerInstruction.SetAllJobTriggersError, CancellationToken.None).ConfigureAwait(false);
continue;
}
var threadPoolRunResult = qsRsrcs.ThreadPool.RunInThread(() => shell.Run(CancellationToken.None));
if (threadPoolRunResult == false)
{
// this case should never happen, as it is indicative of the
// scheduler being shutdown or a bug in the thread pool or
// a thread pool being used concurrently - which the docs
// say not to do...
Log.Error("ThreadPool.RunInThread() returned false!");
await qsRsrcs.JobStore.TriggeredJobComplete(trigger, bndle.JobDetail, SchedulerInstruction.SetAllJobTriggersError, CancellationToken.None).ConfigureAwait(false);
}
}
continue; // while (!halted)
}
}
else // if(availThreadCount > 0)
{
continue;
// while (!halted)
}
DateTimeOffset utcNow = SystemTime.UtcNow();
DateTimeOffset waitTime = utcNow.Add(GetRandomizedIdleWaitTime());
TimeSpan timeUntilContinue = waitTime - utcNow;
lock (sigLock)
{
if (!halted)
{
try
{
// QTZ-336 A job might have been completed in the mean time and we might have
// missed the scheduled changed signal by not waiting for the notify() yet
// Check that before waiting for too long in case this very job needs to be
// scheduled very soon
if (!IsScheduleChanged())
{
Monitor.Wait(sigLock, timeUntilContinue);
}
}
catch (ThreadInterruptedException)
{
}
}
}
}
catch (Exception re)
{
Log.ErrorException("Runtime error occurred in main trigger firing loop.", re);
}
} // while (!halted)
}