public void ShouldReturnAvailableNodes()
{
NodeManager.AddWorkerNode(new Uri("http://localhost:9051/"));
var jobQueueItem = new JobQueueItem
{
JobId = Guid.NewGuid(),
Name = "Name Test",
CreatedBy = "Created By Test",
Serialized = "Serialized Test",
Type = "Type Test"
};
JobManager.AddItemToJobQueue(jobQueueItem);
JobManager.AssignJobToWorkerNodes();
TestHelper.AddDeadNode("http://localhost:9052/");
NodeManager.AddWorkerNode(new Uri("http://localhost:9053/"));
List <Uri> nodes;
using (var sqlConnection = new SqlConnection(ManagerConfiguration.ConnectionString))
{
sqlConnection.Open();
nodes = JobRepositoryCommandExecuter.SelectAllAvailableWorkerNodes(sqlConnection);
}
nodes.Count.Should().Be.EqualTo(1);
nodes.First().Should().Be.EqualTo("http://localhost:9053/");
}
public void ShouldSendTheJobToAnotherNodeIfFirstReturnsConflict()
{
var jobQueueItem = new JobQueueItem
{
Name = "Name Test",
CreatedBy = "Created By Test",
Serialized = "Serialized Test",
Type = "Type Test"
};
var workerNode2 = new WorkerNode
{
Url = new Uri("http://localhost:9051/")
};
NodeRepository.AddWorkerNode(_workerNode);
Thread.Sleep(TimeSpan.FromSeconds(1));
NodeRepository.AddWorkerNode(workerNode2);
ThisNodeIsBusy(workerNode2.Url);
ManagerController.AddItemToJobQueue(jobQueueItem);
while (true)
{
if (JobRepository.GetAllJobs().Count == 1)
{
break;
}
}
JobRepository.GetAllJobs().First().SentToWorkerNodeUri.Should().Be.EqualTo(_workerNode.Url.ToString());
}
public void ShouldJustWarnIfAnotherManagerAlreadySentJob()
{
var jobQueueItem = new JobQueueItem
{
JobId = Guid.NewGuid(),
Name = "Name",
Serialized = "Serialized",
Type = "Type",
CreatedBy = "CreatedBy",
Created = DateTime.Now
};
NodeManager.AddWorkerNode(_workerNode.Url);
using (var sqlConnection = new SqlConnection(ManagerConfiguration.ConnectionString))
{
sqlConnection.Open();
JobRepositoryCommandExecuter.InsertIntoJobQueue(jobQueueItem, sqlConnection);
JobRepositoryCommandExecuter.InsertIntoJob(jobQueueItem, "aNode", sqlConnection);
}
JobManager.AssignJobToWorkerNodes();
var assertSQL = "SELECT COUNT(*) FROM Stardust.Logging WHERE [Level] = 'WARN' AND [Message] like '%It was already assigned, probably by another Manager.%'";
using (var sqlConnection = new SqlConnection(ManagerConfiguration.ConnectionString))
{
using (var getAllJobsCommand = new SqlCommand(assertSQL, sqlConnection))
{
sqlConnection.Open();
var ret = getAllJobsCommand.ExecuteScalar();
ret.Should().Be.EqualTo(1);
}
}
}
public void ShouldHandleNodeDownTemporarilyOnPostAsync()
{
var jobQueueItem = new JobQueueItem
{
JobId = Guid.NewGuid(),
Name = "Name Test",
CreatedBy = "Created By Test",
Serialized = "Serialized Test",
Type = "Type Test"
};
WorkerNodeRepository.AddWorkerNode(_workerNode);
JobManager.AddItemToJobQueue(jobQueueItem);
FakeHttpSender.FailPostAsync = true;
JobManager.AssignJobToWorkerNodes();
JobRepository.GetAllItemsInJobQueue().Count.Should().Be(1);
JobRepository.GetAllJobs().Count.Should().Be(0);
Logger.InfoMessages.Any(l => l.Contains("The remote name could not be resolved")).Should().Be.True();
Logger.ErrorMessages.Should().Be.Empty();
Logger.InfoMessages.Clear();
Logger.ErrorMessages.Clear();
FakeHttpSender.FailPostAsync = false;
JobManager.AssignJobToWorkerNodes();
JobRepository.GetAllItemsInJobQueue().Count.Should().Be(0);
JobRepository.GetAllJobs().Count.Should().Be(1);
}
public void ShouldBeAbleToRequeueDeadJob()
{
var jobQueueItem = new JobQueueItem
{
JobId = Guid.NewGuid(),
Name = "Name Test",
CreatedBy = "Created By Test",
Serialized = "Serialized Test",
Type = "Type Test"
};
WorkerNodeRepository.AddWorkerNode(_workerNode);
JobManager.AddItemToJobQueue(jobQueueItem);
JobManager.AssignJobToWorkerNodes();
var job = JobManager.GetJobByJobId(jobQueueItem.JobId);
job.Satisfy(job1 => job1.Started != null);
job.Satisfy(job1 => job1.Ended == null);
NodeManager.RequeueJobsThatDidNotFinishedByWorkerNodeUri(job.SentToWorkerNodeUri);
JobRepository.GetAllJobs().Count.Should().Be(0);
JobRepository.GetAllItemsInJobQueue().Count.Should().Be(1);
}
public void duplicated_queue_item_should_start_queue_if_its_not_running()
{
IEnumerable <IJob> BaseFakeJobs = new List <IJob> {
fakeJob
};
Mocker.SetConstant(BaseFakeJobs);
var stuckQueueItem = new JobQueueItem
{
JobType = fakeJob.GetType(),
Options = new { TargetId = 12 }
};
//Act
var jobProvider = Mocker.Resolve <JobProvider>();
jobProvider.Initialize();
jobProvider.Queue.Add(stuckQueueItem);
WaitForQueue();
jobProvider.QueueJob(stuckQueueItem.JobType, stuckQueueItem.Options);
WaitForQueue();
//Assert
fakeJob.ExecutionCount.Should().Be(1);
}
public IHttpActionResult AddItemToJobQueue([FromBody] JobQueueItem jobQueueItem)
{
jobQueueItem.JobId = Guid.NewGuid();
var isValidRequest = _validator.ValidateObject(jobQueueItem);
if (!isValidRequest.Success)
{
return(BadRequest(isValidRequest.Message));
}
_jobManager.AddItemToJobQueue(jobQueueItem);
var msg = $"{WhoAmI(Request)} : New job received from client ( jobId, jobName ) : ( {jobQueueItem.JobId}, {jobQueueItem.Name} )";
this.Log().InfoWithLineNumber(msg);
Task.Factory.StartNew(() =>
{
_jobManager.AssignJobToWorkerNodes();
});
return(Ok(jobQueueItem.JobId));
}
public void ShouldNotBeAbleToSendAnotherQueuedJob_WhenCheckPolicy_AndHaveNotEnoughWorkerNode()
{
var jobQueueItem = new JobQueueItem
{
JobId = Guid.NewGuid(),
Name = "Name Test",
CreatedBy = "Created By Test",
Serialized = "Serialized Test",
Type = "Type Test",
Policy = HalfNodesAffinityPolicy.PolicyName
};
var jobQueueItem2 = new JobQueueItem
{
JobId = Guid.NewGuid(),
Name = "Name Test",
CreatedBy = "Created By Test",
Serialized = "Serialized Test",
Type = "Type Test",
Policy = HalfNodesAffinityPolicy.PolicyName
};
WorkerNodeRepository.AddWorkerNode(_workerNode);
WorkerNodeRepository.AddWorkerNode(_workerNode2);
JobManager.AddItemToJobQueue(jobQueueItem);
JobManager.AddItemToJobQueue(jobQueueItem2);
JobManager.AssignJobToWorkerNodes();
JobRepository.GetAllItemsInJobQueue().Count.Should().Be(1);
JobRepository.GetAllJobs().Count.Should().Be(1);
}
public static List <JobQueueItem> GenerateFailingJobParamsRequests(int numberOfJobRequests)
{
List <JobQueueItem> requestModels = null;
if (numberOfJobRequests > 0)
{
requestModels = new List <JobQueueItem>();
for (var i = 1; i <= numberOfJobRequests; i++)
{
var failingJobParams = new FailingJobParams("Error message " + i);
var failingJobParamsJson = JsonConvert.SerializeObject(failingJobParams);
var job = new JobQueueItem
{
Name = "Job Name " + i,
Serialized = failingJobParamsJson,
Type = "NodeTest.JobHandlers.FailingJobParams",
CreatedBy = "Test"
};
requestModels.Add(job);
}
}
return(requestModels);
}
private async Task <string> AddJobAsync(JobQueueItem jobQueueItem)
{
var result = HttpSender.PostAsync(MangerUriBuilder.GetAddToJobQueueUri(), jobQueueItem).Result;
var content = await result.Content.ReadAsStringAsync();
return(content);
}
public void ShouldBeAbleToCancelJobIfStarted()
{
var jobQueueItem = new JobQueueItem
{
JobId = Guid.NewGuid(),
Name = "Name Test",
CreatedBy = "Created By Test",
Serialized = "Serialized Test",
Type = "Type Test"
};
WorkerNodeRepository.AddWorkerNode(_workerNode);
JobManager.AddItemToJobQueue(jobQueueItem);
JobManager.AssignJobToWorkerNodes();
var job = JobManager.GetJobByJobId(jobQueueItem.JobId);
job.Satisfy(job1 => job1.Started != null);
JobManager.CancelJobByJobId(jobQueueItem.JobId);
job = JobManager.GetJobByJobId(jobQueueItem.JobId);
job.Satisfy(job1 => job1.Result.StartsWith("cancel", StringComparison.InvariantCultureIgnoreCase));
JobRepository.GetAllJobs().Count.Should().Be(1);
}
public Guid AddJob(JobQueueItem jobQueueItem)
{
string jobIdString = AddJobAsync(jobQueueItem).Result;
Guid jobId = new Guid(JsonConvert.DeserializeObject <string>(jobIdString));
return(jobId);
}
public void InsertIntoJobQueue(JobQueueItem jobQueueItem, SqlConnection sqlConnection, SqlTransaction sqlTransaction = null)
{
const string insertIntoJobQueueCommandText = @"INSERT INTO [Stardust].[JobQueue]
([JobId],
[Name],
[Type],
[Serialized],
[CreatedBy],
[Created],
[Policy])
VALUES
(@JobId,
@Name,
@Type,
@Serialized,
@CreatedBy,
@Created,
@Policy)" ;
using (var insertIntojobQueueCommand = new SqlCommand(insertIntoJobQueueCommandText, sqlConnection, sqlTransaction))
{
insertIntojobQueueCommand.Parameters.AddWithValue("@JobId", jobQueueItem.JobId);
insertIntojobQueueCommand.Parameters.AddWithValue("@Name", jobQueueItem.Name);
insertIntojobQueueCommand.Parameters.AddWithValue("@Type", jobQueueItem.Type);
insertIntojobQueueCommand.Parameters.AddWithValue("@Serialized", jobQueueItem.Serialized);
insertIntojobQueueCommand.Parameters.AddWithValue("@Created", DateTime.UtcNow);
insertIntojobQueueCommand.Parameters.AddWithValue("@CreatedBy", jobQueueItem.CreatedBy);
insertIntojobQueueCommand.Parameters.AddWithValue("@Policy", jobQueueItem.Policy);
insertIntojobQueueCommand.ExecuteNonQueryWithRetry(_retryPolicy);
}
}
private void Execute(JobQueueItem queueItem)
{
var jobImplementation = _jobs.SingleOrDefault(t => t.GetType() == queueItem.JobType);
if (jobImplementation == null)
{
logger.Error("Unable to locate implementation for '{0}'. Make sure it is properly registered.", queueItem.JobType);
return;
}
var settings = All().Where(j => j.TypeName == queueItem.JobType.ToString()).Single();
using (_notification = new ProgressNotification(jobImplementation.Name))
{
try
{
logger.Debug("Starting {0}. Last execution {1}", queueItem, settings.LastExecution);
var sw = Stopwatch.StartNew();
_notificationProvider.Register(_notification);
jobImplementation.Start(_notification, queueItem.Options);
_notification.Status = ProgressNotificationStatus.Completed;
settings.LastExecution = DateTime.Now;
settings.Success = true;
sw.Stop();
logger.Debug("Job {0} successfully completed in {1:0}.{2} seconds.", queueItem, sw.Elapsed.TotalSeconds, sw.Elapsed.Milliseconds / 100,
sw.Elapsed.Seconds);
}
catch (ThreadAbortException)
{
throw;
}
catch (Exception e)
{
logger.ErrorException("An error has occurred while executing job [" + jobImplementation.Name + "].", e);
_notification.Status = ProgressNotificationStatus.Failed;
_notification.CurrentMessage = jobImplementation.Name + " Failed.";
settings.LastExecution = DateTime.Now;
settings.Success = false;
}
}
//Only update last execution status if was triggered by the scheduler
if (queueItem.Options == null)
{
SaveDefinition(settings);
}
}
private void ProcessQueue()
{
try
{
do
{
using (NestedDiagnosticsContext.Push(Guid.NewGuid().ToString()))
{
try
{
JobQueueItem job = null;
lock (Queue)
{
if (Queue.Count != 0)
{
job = Queue.OrderBy(c => c.Source).First();
logger.Trace("Popping {0} from the queue.", job);
Queue.Remove(job);
}
}
if (job != null)
{
Execute(job);
}
}
catch (ThreadAbortException)
{
throw;
}
catch (Exception e)
{
logger.FatalException("An error has occurred while executing job.", e);
}
}
} while (Queue.Count != 0);
}
catch (ThreadAbortException e)
{
logger.Warn(e.Message);
}
catch (Exception e)
{
logger.ErrorException("Error has occurred in queue processor thread", e);
}
finally
{
StopWatch.Stop();
logger.Trace("Finished processing jobs in the queue.");
}
}
public void ShouldReturnBadRequestIfJobQueueItemHasInvalidCreatedByValue()
{
var jobQueueItem = new JobQueueItem
{
Name = "Name",
Serialized = "Serialized",
Type = "Type",
CreatedBy = string.Empty
};
var response = ManagerController.AddItemToJobQueue(jobQueueItem);
Assert.IsInstanceOf(typeof(BadRequestErrorMessageResult), response);
}
public void ShouldBeAbleToPersistNewJobQueueItem()
{
var jobQueueItem = new JobQueueItem
{
Name = "Name Test",
CreatedBy = "Created By Test",
Serialized = "Serialized Test",
Type = "Type Test"
};
ManagerController.AddItemToJobQueue(jobQueueItem);
JobRepository.GetAllItemsInJobQueue().Count.Should().Be.EqualTo(1);
}
/// <summary>
/// Long running task via BackgroundService
/// </summary>
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
JobQueueItem jobQueueItem = null;
try
{
// wait for the queue to signal there is something that needs to be done
await _signal.WaitAsync(stoppingToken).ConfigureAwait(false);
// dequeue the item
jobQueueItem = _queue.TryDequeue(out var workItem) ? workItem : null;
if (jobQueueItem != null)
{
// put the job in to a "processing" state
await _jobStatusService.UpdateJobStatusAsync(
jobQueueItem.JobId, JobStatus.Processing).ConfigureAwait(false);
// the heavy lifting is done here...
var result = await _workService.DoWorkAsync(
jobQueueItem.JobId, jobQueueItem.JobParameters,
stoppingToken).ConfigureAwait(false);
// store the result of the work and set the status to "finished"
await _jobStatusService.StoreJobResultAsync(
jobQueueItem.JobId, result, JobStatus.Success).ConfigureAwait(false);
}
}
catch (TaskCanceledException)
{
break;
}
catch (Exception ex)
{
try
{
// something went wrong. Put the job in to an errored state and continue on
await _jobStatusService.StoreJobResultAsync(jobQueueItem.JobId, new JobResultModel
{
Exception = new JobExceptionModel(ex)
}, JobStatus.Errored).ConfigureAwait(false);
}
catch (Exception)
{
// TODO: log this
}
}
}
}
public void ShouldReturnBadRequestIfJobRequestModelNameIsNull()
{
var jobQueueItem = new JobQueueItem
{
Name = null,
Serialized = "Serialized",
Type = "Type",
CreatedBy = "UserName"
};
var response = ManagerController.AddItemToJobQueue(jobQueueItem);
Assert.IsInstanceOf(typeof(BadRequestErrorMessageResult), response);
}
public void ShouldBeAbleToAddToJobQueue()
{
var jobQueueItem = new JobQueueItem
{
JobId = Guid.NewGuid(),
Name = "Name Test",
CreatedBy = "Created By Test",
Serialized = "Serialized Test",
Type = "Type Test"
};
JobManager.AddItemToJobQueue(jobQueueItem);
JobRepository.GetAllItemsInJobQueue().Count.Should().Be(1);
}
private static JobQueueItem CreateJobQueueItemFromSqlDataReader(SqlDataReader sqlDataReader)
{
var jobQueueItem = new JobQueueItem
{
JobId = sqlDataReader.GetGuid(sqlDataReader.GetOrdinal("JobId")),
Name = sqlDataReader.GetNullableString(sqlDataReader.GetOrdinal("Name")),
Serialized = sqlDataReader.GetNullableString(sqlDataReader.GetOrdinal("Serialized")).Replace(@"\", @""),
Type = sqlDataReader.GetNullableString(sqlDataReader.GetOrdinal("Type")),
CreatedBy = sqlDataReader.GetNullableString(sqlDataReader.GetOrdinal("CreatedBy")),
Created = sqlDataReader.GetDateTime(sqlDataReader.GetOrdinal("Created")),
Policy = sqlDataReader.GetNullableString(sqlDataReader.GetOrdinal("Policy"))
};
return(jobQueueItem);
}
public void AddItemToJobQueue(JobQueueItem jobQueueItem)
{
try
{
using (var sqlConnection = new SqlConnection(_connectionString))
{
sqlConnection.OpenWithRetry(_retryPolicy);
_jobRepositoryCommandExecuter.InsertIntoJobQueue(jobQueueItem, sqlConnection);
}
}
catch (Exception exp)
{
this.Log().ErrorWithLineNumber(exp.Message, exp);
throw;
}
}
public void ShouldDeleteJobFromJobQueueIfNotAssignedToWorkerNode()
{
var jobQueueItem = new JobQueueItem
{
JobId = Guid.NewGuid(),
Name = "Name Test",
CreatedBy = "Created By Test",
Serialized = "Serialized Test",
Type = "Type Test"
};
JobManager.AddItemToJobQueue(jobQueueItem);
JobManager.CancelJobByJobId(jobQueueItem.JobId);
JobRepository.GetAllItemsInJobQueue().Count.Should().Be(0);
}
public void ShouldDoNothing_WhenNoWorkerNodeExists_AndJobQueueHasItems()
{
var jobQueueItem = new JobQueueItem
{
JobId = Guid.NewGuid(),
Name = "Name Test",
CreatedBy = "Created By Test",
Serialized = "Serialized Test",
Type = "Type Test"
};
JobManager.AddItemToJobQueue(jobQueueItem);
JobManager.AssignJobToWorkerNodes();
FakeHttpSender.CallToWorkerNodes.Should().Be.Empty();
}
public bool CheckPolicy(JobQueueItem jobQueueItem, List <Job> allExecutingJobs, int countOfAliveNodes)
{
ManagerLogger.Info($"Checking HalfNodesAffinityPolicy for job {jobQueueItem.JobId}");
var totalNumberOfNodesCanRun = countOfAliveNodes / 2;
if (jobQueueItem.Policy != PolicyName)
{
return(true);
}
var isSatisfied = allExecutingJobs.Count(x => x.Policy == PolicyName) < totalNumberOfNodesCanRun;
if (!isSatisfied)
{
ManagerLogger.Info($"Failed HalfNodesAffinityPolicy for job {jobQueueItem.JobId}");
}
return(isSatisfied);
}
public void ShouldReturnIdOfPersistedJob()
{
var jobQueueItem = new JobQueueItem
{
Name = "Name Test",
CreatedBy = "Created By Test",
Serialized = "Serialized Test",
Type = "Type Test"
};
IHttpActionResult actionResult =
ManagerController.AddItemToJobQueue(jobQueueItem);
var okNegotiatedContentResult = actionResult as OkNegotiatedContentResult <Guid>;
var jobId = okNegotiatedContentResult.Content;
jobId.Satisfy(guid => guid != Guid.Empty);
}
public void ShouldAssignANewIdForANewQueueItem()
{
Guid oldGuid = Guid.NewGuid();
var jobQueueItem = new JobQueueItem
{
JobId = oldGuid,
Name = "Name",
Serialized = "Serialized",
Type = "Type",
CreatedBy = "CreatedBy"
};
ManagerController.AddItemToJobQueue(jobQueueItem);
var queuedItems = JobRepository.GetAllItemsInJobQueue();
queuedItems[0].JobId.Should().Not.Be.EqualTo(oldGuid);
}
public void InsertIntoJob(JobQueueItem jobQueueItem, string sentToWorkerNodeUri, SqlConnection sqlConnection, SqlTransaction sqlTransaction = null)
{
const string insertIntoJobCommandText = @"INSERT INTO [Stardust].[Job]
([JobId]
,[Name]
,[Created]
,[CreatedBy]
,[Started]
,[Ended]
,[Serialized]
,[Type]
,[SentToWorkerNodeUri]
,[Result]
,[Policy])
VALUES
(@JobId
,@Name
,@Created
,@CreatedBy
,@Started
,@Ended
,@Serialized
,@Type
,@SentToWorkerNodeUri
,@Result
,@Policy)" ;
using (var insertIntoJobCommand = new SqlCommand(insertIntoJobCommandText, sqlConnection, sqlTransaction))
{
insertIntoJobCommand.Parameters.AddWithValue("@Result", DBNull.Value);
insertIntoJobCommand.Parameters.AddWithValue("@JobId", jobQueueItem.JobId);
insertIntoJobCommand.Parameters.AddWithValue("@Name", jobQueueItem.Name);
insertIntoJobCommand.Parameters.AddWithValue("@Type", jobQueueItem.Type);
insertIntoJobCommand.Parameters.AddWithValue("@Serialized", jobQueueItem.Serialized);
insertIntoJobCommand.Parameters.AddWithValue("@Started", DateTime.UtcNow);
insertIntoJobCommand.Parameters.AddWithValue("@SentToWorkerNodeUri", sentToWorkerNodeUri);
insertIntoJobCommand.Parameters.AddWithValue("@CreatedBy", jobQueueItem.CreatedBy);
insertIntoJobCommand.Parameters.AddWithValue("@Created", jobQueueItem.Created);
insertIntoJobCommand.Parameters.AddWithValue("@Ended", DBNull.Value);
insertIntoJobCommand.Parameters.AddWithValue("@Policy", jobQueueItem.Policy ?? (object)DBNull.Value);
insertIntoJobCommand.ExecuteNonQueryWithRetry(_retryPolicy);
}
}
public void ShouldBeAbleToSendAQueuedJob()
{
var jobQueueItem = new JobQueueItem
{
JobId = Guid.NewGuid(),
Name = "Name Test",
CreatedBy = "Created By Test",
Serialized = "Serialized Test",
Type = "Type Test"
};
WorkerNodeRepository.AddWorkerNode(_workerNode);
JobManager.AddItemToJobQueue(jobQueueItem);
JobManager.AssignJobToWorkerNodes();
JobRepository.GetAllItemsInJobQueue().Count.Should().Be(0);
JobRepository.GetAllJobs().Count.Should().Be(1);
}
public void RequeueJobThatDidNotEndByWorkerNodeUri(string workerNodeUri)
{
try
{
lock (_requeueJobLock)
{
using (var sqlConnection = new SqlConnection(_connectionString))
{
sqlConnection.OpenWithRetry(_retryPolicy);
using (var sqlTransaction = sqlConnection.BeginTransaction())
{
var job = _jobRepositoryCommandExecuter.SelectExecutingJob(workerNodeUri, sqlConnection, sqlTransaction);
if (job == null)
{
return;
}
var jobQueueItem = new JobQueueItem
{
Created = job.Created,
CreatedBy = job.CreatedBy,
JobId = job.JobId,
Serialized = job.Serialized,
Name = job.Name,
Type = job.Type,
Policy = job.Policy
};
_jobRepositoryCommandExecuter.InsertIntoJobQueue(jobQueueItem, sqlConnection, sqlTransaction);
_jobRepositoryCommandExecuter.DeleteJob(jobQueueItem.JobId, sqlConnection, sqlTransaction);
sqlTransaction.Commit();
}
}
}
}
catch (Exception exp)
{
this.Log().ErrorWithLineNumber(exp.Message, exp);
throw;
}
}
public void duplicated_queue_item_should_start_queue_if_its_not_running()
{
IList<IJob> BaseFakeJobs = new List<IJob> { fakeJob };
Mocker.SetConstant(BaseFakeJobs);
var stuckQueueItem = new JobQueueItem
{
JobType = fakeJob.GetType(),
TargetId = 12,
SecondaryTargetId = 0
};
//Act
var jobProvider = Mocker.Resolve<JobProvider>();
jobProvider.Initialize();
jobProvider.Queue.Add(stuckQueueItem);
WaitForQueue();
jobProvider.QueueJob(stuckQueueItem.JobType, stuckQueueItem.TargetId, stuckQueueItem.SecondaryTargetId);
WaitForQueue();
//Assert
fakeJob.ExecutionCount.Should().Be(1);
}
private void Execute(JobQueueItem queueItem)
{
var jobImplementation = _jobs.Where(t => t.GetType() == queueItem.JobType).SingleOrDefault();
if (jobImplementation == null)
{
logger.Error("Unable to locate implementation for '{0}'. Make sure it is properly registered.", queueItem.JobType);
return;
}
var settings = All().Where(j => j.TypeName == queueItem.JobType.ToString()).Single();
using (_notification = new ProgressNotification(jobImplementation.Name))
{
try
{
logger.Debug("Starting {0}. Last execution {1}", queueItem, settings.LastExecution);
var sw = Stopwatch.StartNew();
_notificationProvider.Register(_notification);
jobImplementation.Start(_notification, queueItem.TargetId, queueItem.SecondaryTargetId);
_notification.Status = ProgressNotificationStatus.Completed;
settings.LastExecution = DateTime.Now;
settings.Success = true;
sw.Stop();
logger.Debug("Job {0} successfully completed in {1:0}.{2} seconds.", queueItem, sw.Elapsed.TotalSeconds, sw.Elapsed.Milliseconds / 100,
sw.Elapsed.Seconds);
}
catch (ThreadAbortException)
{
throw;
}
catch (Exception e)
{
logger.ErrorException("An error has occurred while executing job [" + jobImplementation.Name + "].", e);
_notification.Status = ProgressNotificationStatus.Failed;
_notification.CurrentMessage = jobImplementation.Name + " Failed.";
settings.LastExecution = DateTime.Now;
settings.Success = false;
}
}
//Only update last execution status if was triggered by the scheduler
if (queueItem.TargetId == 0)
{
SaveDefinition(settings);
}
}
public virtual void QueueJob(Type jobType, int targetId = 0, int secondaryTargetId = 0, JobQueueItem.JobSourceType source = JobQueueItem.JobSourceType.User)
{
var queueItem = new JobQueueItem
{
JobType = jobType,
TargetId = targetId,
SecondaryTargetId = secondaryTargetId,
Source = source
};
logger.Debug("Attempting to queue {0}", queueItem);
lock (executionLock)
{
VerifyThreadTime();
lock (Queue)
{
if (!Queue.Contains(queueItem))
{
Queue.Add(queueItem);
logger.Trace("Job {0} added to the queue. current items in queue: {1}", queueItem, Queue.Count);
}
else
{
logger.Info("{0} already exists in the queue. Skipping. current items in queue: {1}", queueItem, Queue.Count);
}
}
if (_jobThread.IsAlive)
{
logger.Trace("Queue is already running. No need to start it up.");
return;
}
ResetThread();
StopWatch = Stopwatch.StartNew();
_jobThread.Start();
}
}
