public void SchedulerHandlesJobWatcherExceptionByInsertingAnErrorRecoveryDelay()
{
// The mock job watcher will throw on the first GetNextJobToProcess.
// On the second one it will cause us to wake up.
// There should be a total delay at least as big as the error recovery delay.
IJobWatcher mockJobWatcher = Mocks.CreateMock <IJobWatcher>();
Expect.Call(mockJobWatcher.GetNextJobToProcess()).Throw(new Exception("Uh oh!"));
Expect.Call(mockJobWatcher.GetNextJobToProcess()).Do((GetNextJobToProcessDelegate) delegate
{
Wake();
return(null);
});
mockJobWatcher.Dispose();
LastCall.Repeat.AtLeastOnce();
Expect.Call(mockJobStore.CreateJobWatcher(scheduler.Guid)).Return(mockJobWatcher);
Mocks.ReplayAll();
scheduler.ErrorRecoveryDelayInSeconds = 2;
Stopwatch stopWatch = Stopwatch.StartNew();
RunSchedulerUntilWake();
Assert.That(stopWatch.ElapsedMilliseconds, NUnit.Framework.Is.GreaterThanOrEqualTo(2000).And.LessThanOrEqualTo(10000));
}
public void SchedulerHandlesJobWatcherObjectDisposedExceptionByStopping(bool throwSecondExceptionInDispose)
{
// The mock job watcher will throw ObjectDisposedException on the first
// call to GetNextJobToProcess. This should cause the scheduler's job watching
// thread to begin shutting down and eventually call the job watcher's Dispose.
// That will wake us up from sleep so we can verify that the scheduler has stopped
// on its own.
// We also check what happens if a second exception occurs in Dispose during shutdown.
IJobWatcher mockJobWatcher = Mocks.CreateMock <IJobWatcher>();
Expect.Call(mockJobWatcher.GetNextJobToProcess()).Throw(new ObjectDisposedException("Uh oh!"));
mockJobWatcher.Dispose();
LastCall.Do((DisposeDelegate) delegate
{
Wake();
if (throwSecondExceptionInDispose)
{
throw new Exception("Yikes! We're trying to shut down here!");
}
});
Expect.Call(mockJobStore.CreateJobWatcher(scheduler.Guid)).Return(mockJobWatcher);
Mocks.ReplayAll();
scheduler.Start();
WaitUntilWake();
Assert.IsFalse(scheduler.IsRunning, "The scheduler should have stopped itself automatically.");
}
public void RunningJobsAreOrphanedWhenSchedulerRegistrationRefreshFails()
{
Mocks.ReplayAll();
CreateRunningJob("running-job");
SetBrokenConnectionMocking(JobStore, true);
JobStore.SchedulerExpirationTimeInSeconds = 1;
// Allow some time for the expiration time to expire.
Thread.Sleep(3000);
// Now get a new scheduler.
// Its next job up for processing should be the one that we created earlier
// but now it will be Orphaned.
PersistentJobStore newJobStore = CreatePersistentJobStore();
newJobStore.SchedulerExpirationTimeInSeconds = 1;
newJobStore.PollIntervalInSeconds = 1;
IJobWatcher jobWatcher = newJobStore.CreateJobWatcher(Guid.NewGuid());
JobDetails orphanedJob = jobWatcher.GetNextJobToProcess();
Assert.AreEqual("running-job", orphanedJob.JobSpec.Name);
Assert.AreEqual(JobState.Orphaned, orphanedJob.JobState);
Assert.AreEqual(false, orphanedJob.LastJobExecutionDetails.Succeeded);
Assert.IsNotNull(orphanedJob.LastJobExecutionDetails.EndTimeUtc);
}
public void SchedulerHandlesJobStoreConcurrentModificationExceptionByIgnoringTheJob()
{
// The mock job watcher will return job detail on the first GetNextJobToProcess
// but the mock job store will throw ConcurrentModificationException during SaveJobDetails.
// On the second call to GetNextJobToProcess the mock job watcher will cause us to wake up.
// There should be no noticeable delay, particularly not one as big as the error recovery delay.
JobDetails jobDetails = new JobDetails(dummyJobSpec, DateTime.UtcNow);
jobDetails.JobState = JobState.Pending;
Expect.Call(mockTrigger.Schedule(TriggerScheduleCondition.Latch, DateTime.UtcNow, null))
.Constraints(Rhino.Mocks.Constraints.Is.Equal(TriggerScheduleCondition.Latch), Rhino.Mocks.Constraints.Is.Anything(), Rhino.Mocks.Constraints.Is.Null())
.Return(TriggerScheduleAction.Stop);
Expect.Call(mockTrigger.NextFireTimeUtc).Return(new DateTime(1970, 1, 5, 0, 0, 0, DateTimeKind.Utc));
Expect.Call(mockTrigger.NextMisfireThreshold).Return(new TimeSpan(0, 1, 0));
mockJobStore.SaveJobDetails(jobDetails);
LastCall.Throw(new ConcurrentModificationException("Another scheduler grabbed it."));
IJobWatcher mockJobWatcher = Mocks.CreateMock <IJobWatcher>();
Expect.Call(mockJobWatcher.GetNextJobToProcess()).Return(jobDetails);
Expect.Call(mockJobWatcher.GetNextJobToProcess()).Do((GetNextJobToProcessDelegate) delegate
{
Wake();
return(null);
});
mockJobWatcher.Dispose();
LastCall.Repeat.AtLeastOnce();
Expect.Call(mockJobStore.CreateJobWatcher(scheduler.Guid)).Return(mockJobWatcher);
Mocks.ReplayAll();
scheduler.ErrorRecoveryDelayInSeconds = 2;
Stopwatch stopWatch = Stopwatch.StartNew();
RunSchedulerUntilWake();
Assert.Less(stopWatch.ElapsedMilliseconds, 2000);
}
public void SchedulerHandlesJobStoreExceptionByInsertingAnErrorRecoveryDelay()
{
// The mock job watcher will return job detail on the first GetNextJobToProcess
// but the mock job store will throw Expection during SaveJobDetails.
// On the second call to GetNextJobToProcess the mock job watcher will cause us to wake up.
// There should be a total delay at least as big as the error recovery delay.
JobDetails jobDetails = new JobDetails(dummyJobSpec, DateTime.UtcNow);
jobDetails.JobState = JobState.Pending;
Expect.Call(mockTrigger.Schedule(TriggerScheduleCondition.Latch, DateTime.UtcNow, null))
.Constraints(Rhino.Mocks.Constraints.Is.Equal(TriggerScheduleCondition.Latch), Rhino.Mocks.Constraints.Is.Anything(), Rhino.Mocks.Constraints.Is.Null())
.Return(TriggerScheduleAction.Stop);
Expect.Call(mockTrigger.NextFireTimeUtc).Return(new DateTime(1970, 1, 5, 0, 0, 0, DateTimeKind.Utc));
Expect.Call(mockTrigger.NextMisfireThreshold).Return(new TimeSpan(0, 1, 0));
mockJobStore.SaveJobDetails(jobDetails);
LastCall.Throw(new Exception("Uh oh!"));
IJobWatcher mockJobWatcher = Mocks.CreateMock <IJobWatcher>();
Expect.Call(mockJobWatcher.GetNextJobToProcess()).Return(jobDetails);
Expect.Call(mockJobWatcher.GetNextJobToProcess()).Do((GetNextJobToProcessDelegate) delegate
{
Wake();
return(null);
});
mockJobWatcher.Dispose();
LastCall.Repeat.AtLeastOnce();
Expect.Call(mockJobStore.CreateJobWatcher(scheduler.Guid)).Return(mockJobWatcher);
Mocks.ReplayAll();
scheduler.ErrorRecoveryDelayInSeconds = 2;
Stopwatch stopWatch = Stopwatch.StartNew();
RunSchedulerUntilWake();
Assert.That(stopWatch.ElapsedMilliseconds, NUnit.Framework.Is.GreaterThanOrEqualTo(2000).And.LessThanOrEqualTo(10000));
}
public void JobWatcher_UnblocksThreadAndThrowsIfJobStoreIsDisposedAsynchronously()
{
IJobWatcher watcher = jobStore.CreateJobWatcher(SchedulerGuid);
ThreadPool.QueueUserWorkItem(delegate
{
Thread.Sleep(2000);
jobStore.Dispose();
});
// This call blocks until the dispose runs.
watcher.GetNextJobToProcess();
}
public void JobWatcher_UnblocksWhenPendingJobAdded()
{
IJobWatcher watcher = jobStore.CreateJobWatcher(SchedulerGuid);
ThreadPool.QueueUserWorkItem(delegate
{
Thread.Sleep(2000);
CreatePendingJob("pending", DateTime.UtcNow);
});
// Wait for job to become ready.
JobDetails triggered = watcher.GetNextJobToProcess();
// Job should come back pending.
Assert.AreEqual("pending", triggered.JobSpec.Name);
Assert.AreEqual(JobState.Pending, triggered.JobState);
watcher.Dispose();
}
public void JobWatcher_IgnoresExceptionIfDbConnectionFailureOccurs()
{
Mocks.ReplayAll();
SetBrokenConnectionMocking(JobStore, true);
IJobWatcher jobWatcher = JobStore.CreateJobWatcher(SchedulerGuid);
// This could throw an exception but instead we catch and log it then keep going
// until we are disposed.
Stopwatch stopwatch = Stopwatch.StartNew();
ThreadPool.QueueUserWorkItem(delegate
{
Thread.Sleep(2000);
jobWatcher.Dispose();
});
Assert.IsNull(jobWatcher.GetNextJobToProcess());
Assert.That(stopwatch.ElapsedMilliseconds, Is.GreaterThanOrEqualTo(2000).And.LessThanOrEqualTo(4000), "Check that the thread was blocked the whole time.");
}
public void JobWatcher_UnblocksWhenScheduledJobBecomesTriggered()
{
IJobWatcher watcher = jobStore.CreateJobWatcher(SchedulerGuid);
DateTime fireTime = DateTime.UtcNow.AddSeconds(3);
CreateScheduledJob("scheduled", fireTime);
// Wait for job to become ready.
Assert.Less(DateTime.UtcNow, fireTime);
JobDetails triggered = watcher.GetNextJobToProcess();
Assert.GreaterOrEqual(DateTime.UtcNow, fireTime.Subtract(new TimeSpan(0, 0, 0, 0, 500)));
// allow a little imprecision
// Job should come back triggered.
Assert.AreEqual("scheduled", triggered.JobSpec.Name);
Assert.AreEqual(JobState.Triggered, triggered.JobState);
watcher.Dispose();
}
private void WatchTriggeredJobs(object arg)
{
IJobWatcher jobWatcher = (IJobWatcher)arg;
try
{
for (;;)
{
JobDetails jobDetails;
try
{
jobDetails = jobWatcher.GetNextJobToProcess();
if (jobDetails == null)
{
return; // watcher was disposed
}
}
catch (ObjectDisposedException ex)
{
// watcher was disposed not quite so nicely
logger.FatalFormat(ex, "The job store was disposed prematurely. Stopping the scheduler.");
return;
}
catch (Exception ex)
{
logger.ErrorFormat(ex,
"The scheduled job watcher threw an exception. Pausing for '{0}' seconds before resuming job processing.",
errorRecoveryDelayInSeconds);
Thread.Sleep(errorRecoveryDelayInSeconds * 1000);
continue;
}
try
{
ScheduleJob(jobDetails);
}
catch (Exception ex)
{
logger.ErrorFormat(ex,
"The scheduled job processor threw an exception. Pausing for '{0}' seconds before resuming job processing.",
errorRecoveryDelayInSeconds);
Thread.Sleep(errorRecoveryDelayInSeconds * 1000);
}
}
}
finally
{
// Put the scheduler back in the stopped state.
lock (syncRoot)
{
if (currentJobWatcher == jobWatcher)
{
currentJobWatcher = null;
currentJobWatcherThread = null;
}
}
try
{
jobWatcher.Dispose();
}
catch (Exception ex)
{
logger.WarnFormat(ex, "The job watcher threw an exception when disposed. The exception will be ignored.");
}
}
}
public void JobWatcher_YieldsJobsInExpectedSequence()
{
IJobWatcher watcher = jobStore.CreateJobWatcher(SchedulerGuid);
JobDetails orphaned = CreateOrphanedJob("orphaned", new DateTime(1970, 1, 3));
JobDetails pending = CreatePendingJob("pending", new DateTime(1970, 1, 2));
JobDetails triggered = CreateTriggeredJob("triggered", new DateTime(1970, 1, 6));
JobDetails completed = CreateCompletedJob("completed", new DateTime(1970, 1, 1));
JobDetails scheduled = CreateScheduledJob("scheduled", new DateTime(1970, 1, 4));
// Ensure we tolerate a few odd cases where data may not be available like it should.
JobDetails scheduled2 = CreateScheduledJob("scheduled2", new DateTime(1970, 1, 2));
scheduled2.NextTriggerFireTimeUtc = null;
jobStore.SaveJobDetails(scheduled2);
JobDetails completed2 = CreateCompletedJob("completed2", new DateTime(1970, 1, 1));
completed2.LastJobExecutionDetails = null;
jobStore.SaveJobDetails(completed2);
JobDetails orphaned2 = CreateOrphanedJob("orphaned2", new DateTime(1970, 1, 3));
orphaned2.LastJobExecutionDetails.EndTimeUtc = null;
jobStore.SaveJobDetails(orphaned2);
// Populate a table of expected jobs.
List <JobDetails> expectedJobs = new List <JobDetails>(new JobDetails[]
{
orphaned, pending, triggered, completed, scheduled, scheduled2
, completed2, orphaned2
});
// Add in some extra jobs in other states that will not be returned.
CreateRunningJob("running1");
CreateStoppedJob("stopped1");
CreateScheduledJob("scheduled-in-the-future", DateTime.MaxValue);
// Ensure expected jobs are retrieved.
while (expectedJobs.Count != 0)
{
JobDetails actualJob = watcher.GetNextJobToProcess();
JobDetails expectedJob =
expectedJobs.Find(delegate(JobDetails candidate) { return(candidate.JobSpec.Name == actualJob.JobSpec.Name); });
Assert.IsNotNull(expectedJob, "Did expect job {0}", actualJob.JobSpec.Name);
// All expected scheduled jobs will have been triggered.
if (expectedJob.JobState == JobState.Scheduled)
{
expectedJob.JobState = JobState.Triggered;
}
JobAssert.AreEqual(expectedJob, actualJob);
if (expectedJobs.Count == 1)
{
// Ensure same job is returned a second time until its status is changed.
// We wait for Count == 1 because that's the easiest case for which to verify
// this behavior.
JobDetails actualJob2 = watcher.GetNextJobToProcess();
JobAssert.AreEqual(expectedJob, actualJob2);
}
// Change the status to progress.
actualJob.JobState = JobState.Stopped;
jobStore.SaveJobDetails(actualJob);
expectedJobs.Remove(expectedJob);
}
// Ensure next request blocks but is released by the call to dispose.
ThreadPool.QueueUserWorkItem(delegate
{
Thread.Sleep(2);
watcher.Dispose();
});
// This call blocks until the dispose runs.
Assert.IsNull(watcher.GetNextJobToProcess());
}