/// <summary>
/// Initializes a new instance of the Scheduler class.
/// </summary>
/// <param name="workerId">The ID of the worker the scheduler is scheduling for.</param>
/// <param name="applicationName">The name of the application to schedule jobs for.</param>
/// <param name="queueFilters">The queues to process schedules for.</param>
/// <param name="heartbeat">The heartbeat, in seconds, the system is polled for updates. Used to calculate scheduling windows.</param>
/// <param name="repositoryFactory">The repository factory to use when accessing data.</param>
/// <param name="logger">The logger to use when logging messages.</param>
public Scheduler(long workerId, string applicationName, QueueNameFilters queueFilters, int heartbeat, IRepositoryFactory repositoryFactory, ILogger logger)
{
if (workerId < 1)
{
throw new ArgumentOutOfRangeException("workerId", "workerId must be greater than 0.");
}
if (string.IsNullOrEmpty(applicationName))
{
throw new ArgumentNullException("applicationName", "applicationName must contain a value.");
}
if (heartbeat < 1)
{
throw new ArgumentOutOfRangeException("heartbeat", "heartbeat must be greater than 0.");
}
if (repositoryFactory == null)
{
throw new ArgumentNullException("repositoryFactory", "repositoryFactory cannot be null.");
}
if (logger == null)
{
throw new ArgumentNullException("logger", "logger cannot be null.");
}
this.workerId = workerId;
this.applicationName = applicationName;
this.heartbeat = heartbeat;
this.repositoryFactory = repositoryFactory;
this.logger = logger;
this.schedules = new List<ScheduleRecord>();
this.queueFilters = queueFilters ?? QueueNameFilters.Any();
}
/// <summary>
/// Initializes a new instance of the Worker class.
/// </summary>
/// <param name="applicationName">The name of the application the worker belongs to.</param>
/// <param name="id">The ID of the worker in the repository.</param>
/// <param name="name">The name of the worker.</param>
/// <param name="queueFilters">The queue name filters the worker should use while processing queues..</param>
/// <param name="heartbeat">The number of seconds between poll intervals.</param>
/// <param name="schedulerEnabled">A value indicating whether the scheduler is enabled.</param>
/// <param name="repositoryFactory">The repository factory to use when accessing data.</param>
/// <param name="logger">The logger to use when logging messages.</param>
/// <param name="scheduler">The scheduler to use when managing schedules and enqueueing scheduled jobs.</param>
public Worker(string applicationName, long id, string name, QueueNameFilters queueFilters, int heartbeat, bool schedulerEnabled, IRepositoryFactory repositoryFactory, ILogger logger, IScheduler scheduler)
{
if (string.IsNullOrEmpty(applicationName))
{
throw new ArgumentNullException("applicationName", "applicationName must contain a value.");
}
if (id < 1)
{
throw new ArgumentOutOfRangeException("id", "id must be greater than 0.");
}
if (string.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name", "name must contain a value.");
}
if (heartbeat < 1)
{
throw new ArgumentOutOfRangeException("heartbeat", "heartbeat must be greater than 0.");
}
if (repositoryFactory == null)
{
throw new ArgumentNullException("repositoryFactory", "repositoryFactory cannot be null.");
}
if (logger == null)
{
throw new ArgumentNullException("logger", "logger cannot be null.");
}
this.applicationName = applicationName;
this.id = id;
this.name = name;
this.heartbeat = heartbeat * 1000;
this.schedulerEnabled = schedulerEnabled;
this.repositoryFactory = repositoryFactory;
this.logger = logger;
this.queueFilters = queueFilters ?? QueueNameFilters.Any();
this.scheduler = scheduler ?? new Scheduler(id, applicationName, new QueueNameFilters(this.queueFilters.Include, this.queueFilters.Exclude), heartbeat, repositoryFactory, logger);
this.signalThread = new Thread(this.SignalLoop);
this.signalThread.Name = "BlueCollar Signal Thread";
this.signalThread.Start();
}
public void BenchmarkDequeueAndExecute1000Jobs()
{
ManualResetEvent handle = new ManualResetEvent(false);
Queue <QueueRecord> queue = new Queue <QueueRecord>();
TestJob job = new TestJob()
{
SleepDuration = 10
};
string typeName = JobSerializer.GetTypeName(job.GetType());
for (int i = 0; i < 1000; i++)
{
job.Id = Guid.NewGuid();
queue.Enqueue(
new QueueRecord()
{
Id = i + 1,
ApplicationName = BlueCollarSection.Section.ApplicationName,
Data = JobSerializer.Serialize(job),
JobName = job.Name,
JobType = typeName,
QueuedOn = DateTime.UtcNow,
QueueName = "*",
TryNumber = 1
});
}
SignalsRecord signals = new SignalsRecord()
{
QueueNames = "*", WorkerSignal = WorkerSignal.None, WorkingSignal = WorkingSignal.None
};
var transaction = new Mock <IDbTransaction>();
var repository = new Mock <IRepository>();
repository.Setup(r => r.BeginTransaction()).Returns(transaction.Object);
repository.Setup(r => r.BeginTransaction(It.IsAny <IsolationLevel>())).Returns(transaction.Object);
repository.Setup(r => r.CreateWorking(It.IsAny <WorkingRecord>(), It.IsAny <IDbTransaction>())).Returns((WorkingRecord r, IDbTransaction t) => { r.Id = 1; return(r); });
repository.Setup(r => r.GetWorkingSignals(It.IsAny <long>(), It.IsAny <long?>(), It.IsAny <IDbTransaction>())).Returns(signals);
repository.Setup(r => r.GetQueued(It.IsAny <string>(), It.IsAny <QueueNameFilters>(), It.IsAny <DateTime>(), It.IsAny <IDbTransaction>()))
.Returns(
() =>
{
var r = queue.Dequeue();
if (queue.Count == 0)
{
handle.Set();
}
return(r);
});
var factory = new Mock <IRepositoryFactory>();
factory.Setup(f => f.Create()).Returns(repository.Object);
var logger = new Mock <ILogger>();
Stopwatch stopwatch = new Stopwatch();
using (Worker worker = new Worker(BlueCollarSection.Section.ApplicationName, 1, "Test Worker", QueueNameFilters.Any(), 1, false, factory.Object, logger.Object))
{
stopwatch.Start();
worker.Start();
handle.WaitOne();
worker.Stop(false);
stopwatch.Stop();
}
this.TestContext.WriteLine("1,000 jobs with 10ms execution times were dequeued and executed in {0:N3}s.", stopwatch.Elapsed.TotalSeconds);
}
/// <summary>
/// Gets a queued record for the given application and queue names.
/// </summary>
/// <param name="applicationName">The name of the application to get the queued record for.</param>
/// <param name="queueFilters">The queue filters to use when filtering the queues to read from.</param>
/// <param name="queuedBefore">The date to filter on.</param>
/// <param name="transaction">The transaction to use, if applicable.</param>
/// <returns>A queued record, or null if none was found.</returns>
public QueueRecord GetQueued(string applicationName, QueueNameFilters queueFilters, DateTime queuedBefore, IDbTransaction transaction)
{
throw new NotImplementedException();
}
/// <summary>
/// Gets a queued record for the given application and queue names.
/// </summary>
/// <param name="applicationName">The name of the application to get the queued record for.</param>
/// <param name="queueFilters">The queue filters to use when filtering the queues to read from.</param>
/// <param name="queuedBefore">The date to filter on.</param>
/// <param name="transaction">The transaction to use, if applicable.</param>
/// <returns>A queued record, or null if none was found.</returns>
public QueueRecord GetQueued(string applicationName, QueueNameFilters queueFilters, DateTime queuedBefore, IDbTransaction transaction)
{
throw new NotImplementedException();
}
public QueueRecord GetQueued(string applicationName, QueueNameFilters queueFilters, DateTime queuedBefore, IDbTransaction transaction)
{
StringBuilder sb = new StringBuilder(
@"SELECT TOP 1 *
FROM [BlueCollarQueue]
WHERE
[ApplicationName] = @ApplicationName
AND [QueuedOn] <= @QueuedBefore");
if (!queueFilters.IncludesAllQueues)
{
if (queueFilters.Include.Count() > 0)
{
sb.Append("\n AND [QueueName] IN @IncludeQueueNames");
}
if (queueFilters.Exclude.Count() > 0)
{
sb.Append("\n AND");
sb.Append("\n (");
sb.Append("\n [QueueName] IS NULL");
sb.Append("\n OR [QueueName] NOT IN @ExcludeQueueNames");
sb.Append("\n )");
}
}
sb.Append("\n");
sb.Append(
@"ORDER BY [QueuedOn] ASC;");
return this.connection.Query<QueueRecord>(
sb.ToString(),
new
{
ApplicationName = applicationName,
ExcludeQueueNames = queueFilters.Exclude.ToArray(),
IncludeQueueNames = queueFilters.Include.ToArray(),
QueuedBefore = queuedBefore
},
transaction,
true,
null,
null).FirstOrDefault();
}
internal void SignalLoop()
{
while (true)
{
Thread.Sleep(this.heartbeat.Randomize());
try
{
// Prun any orphans that were abandoned for any reason.
this.PruneOrphans();
WorkerStatus status;
long? recordId = null;
lock (this.statusLocker)
{
status = this.Status;
lock (this.runLocker)
{
if (status == WorkerStatus.Working && this.currentRecord != null)
{
recordId = this.currentRecord.Id;
}
}
}
SignalsRecord signals;
// Load the current signals from the repository.
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction())
{
signals = repository.GetWorkingSignals(this.id, recordId, transaction);
if (signals != null
&& (signals.WorkerSignal != WorkerSignal.None
|| signals.WorkingSignal != WorkingSignal.None))
{
repository.ClearWorkingSignalPair(this.id, recordId, transaction);
}
transaction.Commit();
}
}
if (signals != null)
{
bool refreshQueues = false;
// Refresh the queues we're processing.
lock (this.runLocker)
{
QueueNameFilters filters = QueueNameFilters.Parse(signals.QueueNames);
if (!this.queueFilters.Equals(filters))
{
this.queueFilters = QueueNameFilters.Parse(signals.QueueNames);
refreshQueues = true;
}
}
// Perform the signalled operation, if applicable.
if (signals.WorkerSignal == WorkerSignal.Stop)
{
this.logger.Debug("Worker {0} ({1}) received a signal to stop.", this.name, this.id);
this.Stop(false);
}
else
{
if (signals.WorkingSignal == WorkingSignal.Cancel)
{
this.logger.Debug("Worker {0} ({1}) received a signal to cancel its current job.", this.name, this.id);
this.CancelCurrent();
}
if (signals.WorkerSignal == WorkerSignal.Start)
{
this.logger.Debug("Worker {0} ({1}) received a signal to start.", this.name, this.id);
this.Start();
}
lock (this.statusLocker)
{
status = this.Status;
}
if (status == WorkerStatus.Working)
{
if (this.SchedulerEnabled)
{
if (refreshQueues)
{
this.scheduler.QueueFilters = new QueueNameFilters(this.queueFilters.Include, this.queueFilters.Exclude);
}
if (signals.WorkerSignal == WorkerSignal.RefreshSchedules)
{
this.logger.Debug("Worker {0} ({1}) received a signal to refresh its schedules.", this.name, this.id);
this.scheduler.RefreshSchedules();
}
this.scheduler.EnqueueScheduledJobs();
}
}
}
}
else
{
// If no signals were returned, it means that this worker
// is orphaned.
this.Stop(false);
break;
}
}
catch (ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
this.logger.Error(ex, "Exception thrown during the signal loop for worker {0} ({1}).", this.name, this.id);
}
}
}
/// <summary>
/// Initializes a new instance of the Worker class.
/// </summary>
/// <param name="applicationName">The name of the application the worker belongs to.</param>
/// <param name="id">The ID of the worker in the repository.</param>
/// <param name="name">The name of the worker.</param>
/// <param name="queueFilters">The queue name filters the worker should use while processing queues..</param>
/// <param name="heartbeat">The number of seconds between poll intervals.</param>
/// <param name="schedulerEnabled">A value indicating whether the scheduler is enabled.</param>
/// <param name="repositoryFactory">The repository factory to use when accessing data.</param>
/// <param name="logger">The logger to use when logging messages.</param>
public Worker(string applicationName, long id, string name, QueueNameFilters queueFilters, int heartbeat, bool schedulerEnabled, IRepositoryFactory repositoryFactory, ILogger logger)
: this(applicationName, id, name, queueFilters, heartbeat, schedulerEnabled, repositoryFactory, logger, null)
{
}
/// <summary>
/// Dequeues a job to do work on.
/// </summary>
/// <returns>A working record representing the dequeued job.</returns>
internal WorkingRecord DequeueRecord()
{
WorkingRecord working = null;
QueueNameFilters queues = null;
lock (this.runLocker)
{
queues = new QueueNameFilters(this.queueFilters.Include, this.queueFilters.Exclude);
}
using (IRepository repository = this.repositoryFactory.Create())
{
using (IDbTransaction transaction = repository.BeginTransaction(IsolationLevel.RepeatableRead))
{
QueueRecord queued = repository.GetQueued(this.applicationName, queues, DateTime.UtcNow, transaction);
if (queued != null)
{
working = CreateWorking(queued, this.id, queued.ScheduleId, DateTime.UtcNow);
repository.DeleteQueued(queued.Id.Value, transaction);
working = repository.CreateWorking(working, transaction);
this.logger.Info("Worker {0} ({1}) dequeued '{2}'.", this.name, this.id, queued.JobName);
}
transaction.Commit();
}
}
return working;
}
public void SchedulerRefreshSchedules()
{
ScheduleRecord schedule = new ScheduleRecord()
{
ApplicationName = BlueCollarSection.Section.ApplicationName,
Enabled = true,
Id = 1,
Name = "Test",
QueueName = "*",
RepeatType = ScheduleRepeatType.Days,
RepeatValue = 1,
StartOn = DateTime.UtcNow.FloorWithSeconds()
};
ScheduledJobRecord scheduledJob = new ScheduledJobRecord()
{
Data = @"{""SleepDuration"":1000}",
Id = 1,
JobType = JobSerializer.GetTypeName(typeof(TestJob)),
Schedule = schedule,
ScheduleId = 1
};
schedule.ScheduledJobs.Add(scheduledJob);
var transaction = new Mock <IDbTransaction>();
var repository = new Mock <IRepository>();
repository.Setup(r => r.BeginTransaction()).Returns(transaction.Object);
repository.Setup(r => r.GetSchedules(BlueCollarSection.Section.ApplicationName, It.IsAny <IDbTransaction>())).Returns(new ScheduleRecord[] { schedule });
var factory = new Mock <IRepositoryFactory>();
factory.Setup(f => f.Create()).Returns(repository.Object);
var logger = new Mock <ILogger>();
Scheduler scheduler = new Scheduler(1, BlueCollarSection.Section.ApplicationName, QueueNameFilters.Any(), 1, factory.Object, logger.Object);
Assert.AreEqual(0, scheduler.Schedules.Count());
scheduler.RefreshSchedules();
Assert.AreEqual(1, scheduler.Schedules.Count());
}
