public async Task QueueType_CreateCustome_Success()
{
//create sample job with custom queue
Nebula.RegisterJobQueue(typeof(FirstJobQueue <>), QueueTypes.FirstJobQueue);
var jobManager = Nebula.GetJobManager();
var jobStore = Nebula.ComponentContext.GetComponent <IJobStore>();
var jobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(
string.Empty, "sample-job", nameof(FirstJobStep), new JobConfigurationData
{
MaxBatchSize = 100,
MaxConcurrentBatchesPerWorker = 5,
IsIndefinite = true,
MaxBlockedSecondsPerCycle = 300,
QueueTypeName = QueueTypes.FirstJobQueue
});
var jobData = await jobStore.LoadFromAnyTenant(jobId);
var jobQueue =
Nebula.JobStepSourceBuilder.BuildJobStepSource <FirstJobStep>(jobData.Configuration.QueueTypeName);
Assert.AreEqual(typeof(FirstJobQueue <FirstJobStep>), jobQueue.GetType());
}
public async Task StartJob_EnqueueOneItem_QueueShouldBeEmpty()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
Nebula.RegisterJobProcessor(typeof(FirstJobProcessor), typeof(FirstJobStep));
var jobManager = Nebula.GetJobManager();
var jobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData {
QueueTypeName = QueueType.InMemory
});
var queue = Nebula.JobStepSourceBuilder.BuildInMemoryJobQueue <FirstJobStep>(jobId);
await queue.Enqueue(new FirstJobStep { Number = 1 });
var initialLength = await queue.GetQueueLength();
await jobManager.StartJob(Tenant.Id, jobId);
var processedLength = await queue.GetQueueLength();
var jobRunnerManager = Nebula.ComponentContext.GetComponent <IJobRunnerManager>();
Assert.IsTrue(jobRunnerManager.IsJobRunnerStarted(jobId));
Assert.AreEqual(1, initialLength);
Assert.AreEqual(0, processedLength);
}
public async Task StopJob_StopStartedJob_ShouldPurgeQueue()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
Nebula.RegisterJobProcessor(typeof(FirstJobProcessor), typeof(FirstJobStep));
var jobManager = Nebula.GetJobManager();
var jobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData
{
QueueTypeName = QueueType.InMemory
});
var queue = Nebula.JobStepSourceBuilder.BuildInMemoryJobQueue <FirstJobStep>(jobId);
await jobManager.StartJob(Tenant.Id, jobId);
await queue.Enqueue(new FirstJobStep { Number = 1 });
var initialQueueLength = await queue.GetQueueLength();
await jobManager.StopJob(Tenant.Id, jobId);
var afterStopQueueLength = await queue.GetQueueLength();
Assert.AreEqual(1, initialQueueLength);
Assert.AreEqual(0, afterStopQueueLength);
}
public async Task StopJob_StopStartedJob_CheckJobState()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
Nebula.RegisterJobProcessor(typeof(FirstJobProcessor), typeof(FirstJobStep));
var jobManager = Nebula.GetJobManager();
var jobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData
{
QueueTypeName = QueueType.InMemory
});
var queue = Nebula.JobStepSourceBuilder.BuildInMemoryJobQueue <FirstJobStep>(jobId);
await queue.Enqueue(new FirstJobStep { Number = 1 });
await jobManager.StartJob(Tenant.Id, jobId);
await jobManager.StopJob(Tenant.Id, jobId);
var jobStore = Nebula.ComponentContext.GetComponent(typeof(IJobStore)) as IJobStore;
var job = await jobStore.Load(Tenant.Id, jobId);
Assert.IsNotNull(job);
Assert.AreEqual(JobState.Stopped, job.Status.State);
}
public async Task PurgeQueue_CustomeJobQueueDefinedInJob_shouldBeEmpty()
{
//create sample job with custom queue
var jobManager = Nebula.GetJobManager();
var jobStore = Nebula.ComponentContext.GetComponent <IJobStore>();
Nebula.RegisterJobQueue(typeof(FirstJobQueue <FirstJobStep>), QueueTypes.FirstJobQueue);
var jobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(
string.Empty, "sample-job", nameof(FirstJobStep), new JobConfigurationData
{
MaxBatchSize = 100,
MaxConcurrentBatchesPerWorker = 5,
IsIndefinite = true,
MaxBlockedSecondsPerCycle = 300,
QueueTypeName = QueueTypes.FirstJobQueue
});
var jobData = await jobStore.LoadFromAnyTenant(jobId);
var jobQueue =
Nebula.JobStepSourceBuilder.BuildJobStepSource <FirstJobStep>(jobData.Configuration.QueueTypeName,
jobData.JobId) as IJobQueue <FirstJobStep>;
await jobQueue.Enqueue(new FirstJobStep());
await jobQueue.Purge();
var queueLeghnt = await jobQueue.GetQueueLength();
Assert.AreEqual(0, queueLeghnt);
}
public async Task StopJob_InCompletePreprocessor_JobActionHasPreprocessorDependencyErrorKey()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
Nebula.RegisterJobProcessor(typeof(FirstJobProcessor), typeof(FirstJobStep));
var jobManager = Nebula.GetJobManager();
var preprocessorJobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData
{
QueueTypeName = QueueType.InMemory
});
var jobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData
{
QueueTypeName = QueueType.InMemory
});
await jobManager.AddPredecessor(Tenant.Id, jobId, preprocessorJobId);
var result = await jobManager.StopJob(Tenant.Id, jobId);
Assert.IsFalse(result.Success);
Assert.AreEqual(ErrorKeys.JobActionHasPreprocessorDependency, result.Errors.FirstOrDefault()?.ErrorKey);
}
protected override void ConfigureNebula()
{
RegisterMockJobStore();
RegisterMockBackgroundTaskScheduler();
RegisterMockRedisManager();
Nebula.RegisterJobQueue(typeof(DelayedJobQueue <>), QueueType.Delayed);
}
protected override void ConfigureNebula()
{
RegisterMockRedisManager();
RegisterMockJobStore();
Nebula.RegisterJobQueue(typeof(FirstJobQueue <FirstJobStep>), nameof(FirstJobStep));
Nebula.RegisterJobQueue(typeof(SecondJobQueue <SecondJobStep>), nameof(SecondJobStep));
}
public void JobProcessor_Instance_Registration_ShouldReturnSameInstances()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
Nebula.RegisterJobProcessor(typeof(TransientProcessor1), typeof(FirstJobStep));
var jobProcessor1 = Nebula.ComponentContext.GetComponent <IJobProcessor <FirstJobStep> >();
var jobProcessor2 = Nebula.ComponentContext.GetComponent <IJobProcessor <FirstJobStep> >();
Assert.AreEqual(jobProcessor2, jobProcessor1);
}
public void Initialize()
{
if (_initialized)
{
return;
}
Nebula.RegisterJobQueue(typeof(FirstJobQueue <FirstJobStep>), nameof(FirstJobStep));
Nebula.RegisterJobQueue(typeof(SecondJobQueue <SecondJobStep>), nameof(SecondJobStep));
_initialized = true;
}
public async Task CreateJob_ZeroThrottledMaxBurstSize_ExceptionThrown()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
var jobManager = Nebula.GetJobManager();
await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData
{
ThrottledMaxBurstSize = JobConfigurationDefaultValues.MinThrottledMaxBurstSize,
QueueTypeName = QueueType.InMemory
});
}
public void JobProcessor_InstanceProvider_ReturnsInterface_ShouldReturnDifferentInstances()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
Nebula.RegisterJobProcessor(() => _sp.GetService <IJobProcessor <FirstJobStep> >(),
typeof(FirstJobStep));
var jobProcessor1 = Nebula.ComponentContext.GetComponent <IJobProcessor <FirstJobStep> >();
var jobProcessor2 = Nebula.ComponentContext.GetComponent <IJobProcessor <FirstJobStep> >();
Assert.AreNotEqual(jobProcessor2, jobProcessor1);
}
public async Task CreateJob_ExpiredJob_ExceptionThrown()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
var jobManager = Nebula.GetJobManager();
await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData
{
ExpiresAt = DateTime.Now.AddDays(-1),
QueueTypeName = QueueType.InMemory
});
}
public async Task CreateJob_jobWithoutJobId_ShouldHaveValue()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
var jobManager = Nebula.GetJobManager();
var jobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData {
QueueTypeName = QueueType.InMemory
});
Assert.IsTrue(!jobId.IsNullOrWhitespace());
}
protected override void ConfigureNebula()
{
Nebula.KafkaConfig = new List <KeyValuePair <string, object> >
{
new KeyValuePair <string, object>("bootstrap.servers", "172.30.3.59:9101"),
new KeyValuePair <string, object>("group.id", "testGroup"),
new KeyValuePair <string, object>("auto.offset.reset", "earliest"),
new KeyValuePair <string, object>("queue.buffering.max.ms", 1),
new KeyValuePair <string, object>("batch.num.messages", 1),
new KeyValuePair <string, object>("fetch.wait.max.ms", 5000)
};
Nebula.RegisterJobQueue(typeof(KafkaJobQueue <>), QueueType.Kafka);
}
public async Task CreateJob_NewJob_QueueExistenceCheck()
{
Nebula.RegisterJobQueue(typeof(FirstJobQueue <FirstJobStep>), QueueTypes.FirstJobQueue);
var jobManager = Nebula.GetJobManager();
await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData {
QueueTypeName = QueueTypes.FirstJobQueue
});
var queue = Nebula.JobStepSourceBuilder.BuildJobStepSource <FirstJobStep>(QueueTypes.FirstJobQueue) as FirstJobQueue <FirstJobStep>;
Assert.IsNotNull(queue);
Assert.IsTrue(queue.QueueExistenceChecked);
}
public async Task CreateJob_MaxBatchSizeRange_LessTahnMax()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
var jobManager = Nebula.GetJobManager();
var jobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData
{
MaxBatchSize = JobConfigurationDefaultValues.MaxBatchSize + 1,
QueueTypeName = QueueType.InMemory
});
var jobStore = Nebula.ComponentContext.GetComponent(typeof(IJobStore)) as IJobStore;
var job = await jobStore?.Load(Tenant.Id, jobId);
Assert.IsTrue(job.Configuration.MaxBatchSize <= JobConfigurationDefaultValues.MaxBatchSize);
}
public async Task StartJob_StartNewJob_ShouldCreateRunner()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
Nebula.RegisterJobProcessor(typeof(FirstJobProcessor), typeof(FirstJobStep));
var jobManager = Nebula.GetJobManager();
var jobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData {
QueueTypeName = QueueType.InMemory
});
await jobManager.StartJob(Tenant.Id, jobId);
var jobRunnerManager = Nebula.ComponentContext.GetComponent <IJobRunnerManager>();
Assert.IsTrue(jobRunnerManager.IsJobRunnerStarted(jobId));
}
public async Task CreateJob_MinConcurrentBatchesPerWorker_MoreThanMin()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
var jobManager = Nebula.GetJobManager();
var jobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData
{
MaxConcurrentBatchesPerWorker = JobConfigurationDefaultValues.MinConcurrentBatchesPerWorker - 1,
QueueTypeName = QueueType.InMemory
});
var jobStore = Nebula.ComponentContext.GetComponent(typeof(IJobStore)) as IJobStore;
var job = await jobStore?.Load(Tenant.Id, jobId);
Assert.IsTrue(job.Configuration.MaxConcurrentBatchesPerWorker >=
JobConfigurationDefaultValues.MinConcurrentBatchesPerWorker);
}
public async Task CreateJob_IdleSecondsToCompletion_MoreThanMin()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
var jobManager = Nebula.GetJobManager();
var jobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData
{
IdleSecondsToCompletion = JobConfigurationDefaultValues.MinIdleSecondsToCompletion - 1,
QueueTypeName = QueueType.InMemory
});
var jobStore = Nebula.ComponentContext.GetComponent(typeof(IJobStore)) as IJobStore;
var job = await jobStore?.Load(Tenant.Id, jobId);
Assert.IsTrue(job.Configuration.IdleSecondsToCompletion >=
JobConfigurationDefaultValues.MinIdleSecondsToCompletion);
}
public async Task CreateJob_DefaultThrottledItemsPerSecond_MoreThanMin()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
var jobManager = Nebula.GetJobManager();
var jobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData
{
ThrottledItemsPerSecond = JobConfigurationDefaultValues.MinThrottledItemsPerSecond - 0.0001,
QueueTypeName = QueueType.InMemory
});
var jobStore = Nebula.ComponentContext.GetComponent(typeof(IJobStore)) as IJobStore;
var job = await jobStore?.Load(Tenant.Id, jobId);
Assert.IsTrue(job.Configuration.ThrottledItemsPerSecond >=
JobConfigurationDefaultValues.MinThrottledItemsPerSecond);
}
public async Task StopJob_IndefiniteJob_InvalidJobActionErrorKey()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
Nebula.RegisterJobProcessor(typeof(FirstJobProcessor), typeof(FirstJobStep));
var jobManager = Nebula.GetJobManager();
var jobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData
{
IsIndefinite = true,
QueueTypeName = QueueType.InMemory
});
var result = await jobManager.StopJob(Tenant.Id, jobId);
Assert.IsFalse(result.Success);
Assert.AreEqual(ErrorKeys.InvalidJobAction, result.Errors.FirstOrDefault()?.ErrorKey);
}
public void JobStepSourceBuilder_GetSameJobQueue_ShouldInitializeOnce()
{
Nebula.RegisterJobQueue(typeof(FirstJobQueue <>), QueueTypes.FirstJobQueue);
var jobId1 = "jobId1";
var queue1 =
Nebula.JobStepSourceBuilder.BuildJobStepSource <FirstJobStep>(QueueTypes.FirstJobQueue, jobId1) as
FirstJobQueue <FirstJobStep>;
var queue2 =
Nebula.JobStepSourceBuilder.BuildJobStepSource <FirstJobStep>(QueueTypes.FirstJobQueue, jobId1) as
FirstJobQueue <FirstJobStep>;
Assert.IsNotNull(queue1);
Assert.IsNotNull(queue2);
Assert.AreEqual(queue1, queue2);
Assert.AreEqual(1, queue1.InitializationCount);
}
public async Task StartJobIfNotStarted_StartJob_StateShouldBeInProgress()
{
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
Nebula.RegisterJobProcessor(typeof(FirstJobProcessor), typeof(FirstJobStep));
var jobManager = Nebula.GetJobManager();
var jobId = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id,
configuration : new JobConfigurationData {
QueueTypeName = QueueType.InMemory
});
await jobManager.StartJobIfNotStarted(Tenant.Id, jobId);
var jobStore = Nebula.ComponentContext.GetComponent(typeof(IJobStore)) as IJobStore;
var JobData = await jobStore.Load(Tenant.Id, jobId);
Assert.IsNotNull(JobData);
Assert.AreEqual(JobState.InProgress, JobData.Status.State);
}
public async Task QueueType_DifferentJobs_DifferentQueues()
{
//create sample job with custom queue
var jobManager = Nebula.GetJobManager();
var jobStore = Nebula.ComponentContext.GetComponent <IJobStore>();
Nebula.RegisterJobQueue(typeof(FirstJobQueue <>), QueueTypes.FirstJobQueue);
Nebula.RegisterJobQueue(typeof(SecondJobQueue <>), QueueTypes.SecondJobQueue);
var firstJob = await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(
string.Empty, "first-job", nameof(FirstJobStep), new JobConfigurationData
{
MaxBatchSize = 100,
MaxConcurrentBatchesPerWorker = 5,
IsIndefinite = true,
MaxBlockedSecondsPerCycle = 300,
QueueTypeName = QueueTypes.FirstJobQueue
});
var secondJob = await jobManager.CreateNewJobOrUpdateDefinition <SecondJobStep>(
string.Empty, "second-job", nameof(SecondJobStep), new JobConfigurationData
{
MaxBatchSize = 100,
MaxConcurrentBatchesPerWorker = 5,
IsIndefinite = true,
MaxBlockedSecondsPerCycle = 300,
QueueTypeName = QueueTypes.SecondJobQueue
});
var firstJobData = await jobStore.LoadFromAnyTenant(firstJob);
var firstJobQueue =
Nebula.GetJobQueue <FirstJobStep>(firstJobData.Configuration.QueueTypeName);
var secondJobData = await jobStore.LoadFromAnyTenant(secondJob);
var secondJobQueue =
Nebula.GetJobQueue <SecondJobStep>(secondJobData.Configuration.QueueTypeName);
Assert.AreEqual(typeof(FirstJobQueue <FirstJobStep>), firstJobQueue.GetType());
Assert.AreEqual(typeof(SecondJobQueue <SecondJobStep>), secondJobQueue.GetType());
}
public async Task CreateJob_NewJob_ShouldExistInStore()
{
var jobId = "testJobId";
var jobName = "testJob";
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <FirstJobStep>), QueueType.InMemory);
var jobManager = Nebula.GetJobManager();
await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id, jobName, jobId,
new JobConfigurationData { QueueTypeName = QueueType.InMemory });
await jobManager.CreateNewJobOrUpdateDefinition <FirstJobStep>(Tenant.Id, "secondJob", "secondJob",
new JobConfigurationData { QueueTypeName = QueueType.InMemory });
var jobStore = Nebula.ComponentContext.GetComponent(typeof(IJobStore)) as IJobStore;
var job = await jobStore?.Load(Tenant.Id, jobId);
Assert.IsNotNull(job);
Assert.AreEqual(jobId, job.JobId);
}
protected override void ConfigureNebula()
{
RegisterMockRedisManager();
Nebula.RegisterJobQueue(typeof(FirstJobQueue <FirstJobStep>), nameof(FirstJobStep));
Nebula.RegisterJobQueue(typeof(SecondJobQueue <SecondJobStep>), nameof(SecondJobStep));
Nebula.RegisterJobQueue(typeof(KafkaJobQueue <>), QueueType.Kafka);
Nebula.RegisterJobQueue(typeof(DelayedJobQueue <>), QueueType.Delayed);
Nebula.RegisterJobQueue(typeof(InMemoryJobQueue <>), QueueType.InMemory);
Nebula.RegisterJobQueue(typeof(RedisJobQueue <>), QueueType.Redis);
Nebula.RegisterJobQueue(typeof(InlineJobQueue <>), QueueType.Inline);
Nebula.RegisterJobQueue(typeof(NullJobQueue <>), QueueType.Null);
Nebula.KafkaConfig = new List <KeyValuePair <string, object> >
{
new KeyValuePair <string, object>("bootstrap.servers", "172.30.3.59:9101"),
new KeyValuePair <string, object>("group.id", "testGroup"),
new KeyValuePair <string, object>("auto.offset.reset", "earliest"),
new KeyValuePair <string, object>("queue.buffering.max.ms", 1),
new KeyValuePair <string, object>("batch.num.messages", 1),
new KeyValuePair <string, object>("fetch.wait.max.ms", 5000)
};
}