public void CreateRequestShouldReturnCancelStatusWhenJobHasStartedAndBeenCanceled() { WaitForNodeToFinishWorking(); var startedTest = DateTime.UtcNow; var manualResetEventSlim = new ManualResetEventSlim(); var checkTablesInManagerDbTimer = new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100); bool sentCancel = false; checkTablesInManagerDbTimer.GetJobItems += (sender, items) => { if (items.Any() && items.All(job => job.Ended == null) && sentCancel == false) { sentCancel = true; HttpRequestManager.CancelJob(items.First().JobId); } if (items.Any() && items.All(job => job.Ended != null)) { manualResetEventSlim.Set(); } }; checkTablesInManagerDbTimer.JobTimer.Start(); var jobQueueItem = JobHelper.GenerateTestJobRequests(1, 60).First(); var jobId = HttpRequestManager.AddJob(jobQueueItem); manualResetEventSlim.Wait(TimeSpan.FromSeconds(10)); Assert.IsTrue(!checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job queue must be empty."); Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "Jobs must have been added."); Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(job => job.Result.StartsWith("Canceled", StringComparison.InvariantCultureIgnoreCase))); checkTablesInManagerDbTimer.Dispose(); var endedTest = DateTime.UtcNow; var description = string.Format("Creates Cancel Job with {0} manager and {1} nodes.", NumberOfManagers, NumberOfNodes); DatabaseHelper.AddPerformanceData(ManagerDbConnectionString, description, startedTest, endedTest); }
public void JobShouldHaveStatusFailedIfFailedTest() { WaitForNodeToFinishWorking(); var startedTest = DateTime.UtcNow; var manualResetEventSlim = new ManualResetEventSlim(); var checkTablesInManagerDbTimer = new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100); checkTablesInManagerDbTimer.GetJobItems += (sender, items) => { if (items.Any() && items.All(job => job.Ended != null)) { if (!manualResetEventSlim.IsSet) { manualResetEventSlim.Set(); } } }; checkTablesInManagerDbTimer.JobTimer.Start(); var failingJobQueueItem = JobHelper.GenerateFailingJobParamsRequests(1).First(); var jobId = HttpRequestManager.AddJob(failingJobQueueItem); manualResetEventSlim.Wait(TimeSpan.FromSeconds(10)); Assert.IsTrue(!checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job queue must be empty."); Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "Jobs must have been added."); Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository. Jobs.Any(job => job.Result.StartsWith("fail", StringComparison.InvariantCultureIgnoreCase))); var endedTest = DateTime.UtcNow; var description = string.Format("Creates FAILED jobs with {0} manager and {1} nodes.", NumberOfManagers, NumberOfNodes); DatabaseHelper.AddPerformanceData(ManagerDbConnectionString, description, startedTest, endedTest); checkTablesInManagerDbTimer.Dispose(); }
public void ShouldBeAbleToCreateManySuccessJobRequestTest() { var startedTest = DateTime.UtcNow; var manualResetEventSlim = new ManualResetEventSlim(); var createNewJobRequests = JobHelper.GenerateTestJobRequests(50, 1); var checkTablesInManagerDbTimer = new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 2000); checkTablesInManagerDbTimer.GetJobItems += (sender, jobs) => { if (jobs.Count == createNewJobRequests.Count && jobs.All(job => job.Started != null && job.Ended != null)) { manualResetEventSlim.Set(); } }; checkTablesInManagerDbTimer.JobTimer.Start(); foreach (var newJobRequest in createNewJobRequests) { var jobId = HttpRequestManager.AddJob(newJobRequest); } manualResetEventSlim.Wait(TimeSpan.FromMinutes(5)); var jobsToAssert = checkTablesInManagerDbTimer.ManagerDbRepository.Jobs; Assert.IsTrue(jobsToAssert.Count == createNewJobRequests.Count); Assert.IsTrue(jobsToAssert.All(job => job.Result.StartsWith("Success", StringComparison.InvariantCultureIgnoreCase))); checkTablesInManagerDbTimer.Dispose(); var endedTest = DateTime.UtcNow; var description = string.Format("Creates {0} FAST = 1 second jobs with {1} manager and {2} nodes.", createNewJobRequests.Count, NumberOfManagers, NumberOfNodes); DatabaseHelper.AddPerformanceData(ManagerDbConnectionString, description, startedTest, endedTest); }
public void JobsShouldJustBeQueuedIfNoNodesTest() { var startedTest = DateTime.UtcNow; var manualResetEventSlim = new ManualResetEventSlim(); var checkTablesInManagerDbTimer = new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100); checkTablesInManagerDbTimer.GetJobQueueItems += (sender, items) => { if (items.Any()) { manualResetEventSlim.Set(); } }; checkTablesInManagerDbTimer.JobQueueTimer.Start(); var jobQueueItem = JobHelper.GenerateTestJobRequests(1, 5).First(); HttpRequestManager.AddJob(jobQueueItem); manualResetEventSlim.Wait(TimeSpan.FromSeconds(30)); //Wait a little time to make sure it is only queued and not sent to node Thread.Sleep(TimeSpan.FromSeconds(5)); Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any()); Assert.IsTrue(!checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any()); checkTablesInManagerDbTimer.Dispose(); var endedTest = DateTime.UtcNow; var description = string.Format("Creates {0} Test Timer jobs with {1} manager and {2} nodes.", 1, NumberOfManagers, NumberOfNodes); DatabaseHelper.AddPerformanceData(ManagerDbConnectionString, description, startedTest, endedTest); }
public void ShouldBeAbleToCreateASuccessJobRequestTest() { WaitForNodeToFinishWorking(); var startedTest = DateTime.UtcNow; var manualResetEventSlim = new ManualResetEventSlim(); var checkTablesInManagerDbTimer = new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 2000); checkTablesInManagerDbTimer.GetJobItems += (sender, items) => { if (items.Any() && items.All(job => job.Started != null && job.Ended != null)) { manualResetEventSlim.Set(); } }; checkTablesInManagerDbTimer.JobTimer.Start(); var jobQueueItem = JobHelper.GenerateTestJobRequests(1, 5).First(); var jobId = HttpRequestManager.AddJob(jobQueueItem); manualResetEventSlim.Wait(TimeSpan.FromSeconds(30)); Assert.IsTrue(!checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Should not be any jobs left in queue."); Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "There should be jobs in jobs table."); Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository. Jobs.All(job => job.Result.StartsWith("success", StringComparison.InvariantCultureIgnoreCase))); checkTablesInManagerDbTimer.Dispose(); var endedTest = DateTime.UtcNow; var description = string.Format("Creates {0} Test Timer jobs with {1} manager and {2} nodes.", 1, NumberOfManagers, NumberOfNodes); DatabaseHelper.AddPerformanceData(ManagerDbConnectionString, description, startedTest, endedTest); }
public void NodeShouldNotGetStuckWhenCancellingOrFailingJobs() { WaitForNodeToFinishWorking(); var jobRepository = new ManagerDbRepository(ManagerDbConnectionString); var jobQueueItemCancel = JobHelper.GenerateTestJobRequests(1, 60).First(); var jobQueueItemFail = JobHelper.GenerateFailingJobParamsRequests(1).First(); var jobQueueItemSuccess = JobHelper.GenerateTestJobRequests(1, 2).First(); var cancelId = HttpRequestManager.AddJob(jobQueueItemCancel); WaitForNodeToStartWorking(); HttpRequestManager.CancelJob(cancelId); WaitForNodeToFinishWorking(); Thread.Sleep(TimeSpan.FromSeconds(2)); HttpRequestManager.AddJob(jobQueueItemFail); Thread.Sleep(TimeSpan.FromSeconds(2)); //might be a risc it ends before we see it is working.. WaitForNodeToFinishWorking(); Thread.Sleep(TimeSpan.FromSeconds(2)); HttpRequestManager.AddJob(jobQueueItemSuccess); WaitForNodeToStartWorking(); WaitForNodeToFinishWorking(); Thread.Sleep(TimeSpan.FromSeconds(2)); var queueItems = jobRepository.JobQueueItems; var jobs = jobRepository.Jobs; Assert.IsTrue(!queueItems.Any(), "Job queue must be empty."); Assert.IsTrue(jobs.Any(), "Jobs must have been added."); Assert.IsTrue(jobs.Any(job => job.Result.StartsWith("Canceled", StringComparison.InvariantCultureIgnoreCase))); Assert.IsTrue(jobs.Any(job => job.Result.StartsWith("fail", StringComparison.InvariantCultureIgnoreCase))); Assert.IsTrue(jobs.Any(job => job.Result.StartsWith("success", StringComparison.InvariantCultureIgnoreCase))); }
public void ShouldHandleNodeDisapperingTemporarily() { var startedTest = DateTime.UtcNow; var numberOfJobs = 20; var waitForJobToFinishEvent = new ManualResetEventSlim(); var waitForNodeToStartEvent = new ManualResetEventSlim(); var waitForAllNodesToFinishAJob = new ManualResetEventSlim(); var checkTablesInManagerDbTimer = new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 300); checkTablesInManagerDbTimer.GetJobItems += (sender, items) => { if (items.Count(job => job.Ended != null) == numberOfJobs) { waitForJobToFinishEvent.Set(); } if (items.Where(job => job.Ended != null).GroupBy(job => job.SentToWorkerNodeUri).Count() == 2) { waitForAllNodesToFinishAJob.Set(); } }; checkTablesInManagerDbTimer.GetWorkerNodes += (sender, nodes) => { if (nodes.Count == 2) { waitForNodeToStartEvent.Set(); } }; checkTablesInManagerDbTimer.JobTimer.Start(); checkTablesInManagerDbTimer.WorkerNodeTimer.Start(); //start second node Task <string> taskStartNewNode = new Task <string>(() => { string res = IntegrationControllerApiHelper.StartNewNode(HttpSender).Result; return(res); }); taskStartNewNode.Start(); waitForNodeToStartEvent.Wait(); var jobQueueItemsBatch1 = JobHelper.GenerateTestJobRequests(numberOfJobs, 1); jobQueueItemsBatch1.ForEach(jobQueueItem => HttpRequestManager.AddJob(jobQueueItem)); var allNodesRunningWithoutTimeout = waitForAllNodesToFinishAJob.Wait(TimeSpan.FromSeconds(60)); Assert.IsTrue(allNodesRunningWithoutTimeout, "Timeout when waiting for all nodes to finish at least one job"); Assert.AreNotEqual(0, checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueCount); Task <string> taskShutDownNode2 = new Task <string>(() => { string res = IntegrationControllerApiHelper.ShutDownNode(HttpSender, "Node2.config").Result; return(res); }); taskShutDownNode2.RunSynchronously(); //restart second node Task <string> taskStartNewNodeRestarted = new Task <string>(() => { string res = IntegrationControllerApiHelper.StartNewNode(HttpSender).Result; return(res); }); taskStartNewNodeRestarted.RunSynchronously(); var jobsFinishedWithoutTimeout = waitForJobToFinishEvent.Wait(TimeSpan.FromSeconds(200)); Assert.IsTrue(jobsFinishedWithoutTimeout, "Timeout on Finishing jobs"); // Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.Count == 3, "There should be two nodes registered"); Assert.IsFalse(checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job queue should be empty."); Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "Job should not be empty."); Assert.AreEqual(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.Count, checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Select(j => j.SentToWorkerNodeUri).Distinct().Count()); checkTablesInManagerDbTimer.Dispose(); var endedTest = DateTime.UtcNow; var description = string.Format("Creates {0} Test Timer jobs with {1} manager and {2} nodes.", numberOfJobs, NumberOfManagers, NumberOfNodes + 2); DatabaseHelper.AddPerformanceData(ManagerDbConnectionString, description, startedTest, endedTest); }
public void ShouldHandleMultipleJobsUsingAllNodesAvailable() { var startedTest = DateTime.UtcNow; var numberOfJobs = 15; var waitForJobToFinishEvent = new ManualResetEventSlim(); var waitForNodeToStartEvent = new ManualResetEventSlim(); var checkTablesInManagerDbTimer = new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100); checkTablesInManagerDbTimer.GetJobItems += (sender, items) => { if (items.Count(j => j.Ended != null) == numberOfJobs) { waitForJobToFinishEvent.Set(); } }; checkTablesInManagerDbTimer.GetWorkerNodes += (sender, nodes) => { if (nodes.Count == 2) { waitForNodeToStartEvent.Set(); } }; checkTablesInManagerDbTimer.JobTimer.Start(); checkTablesInManagerDbTimer.WorkerNodeTimer.Start(); //start second node Task <string> taskStartNewNode = new Task <string>(() => { string res = IntegrationControllerApiHelper.StartNewNode(HttpSender).Result; return(res); }); taskStartNewNode.Start(); waitForNodeToStartEvent.Wait(); var jobQueueItems = JobHelper.GenerateTestJobRequests(numberOfJobs, 1); jobQueueItems.ForEach(jobQueueItem => HttpRequestManager.AddJob(jobQueueItem)); var jobsFinishedWithoutTimeout = waitForJobToFinishEvent.Wait(TimeSpan.FromSeconds(60)); Assert.IsTrue(jobsFinishedWithoutTimeout, "Timeout on Finishing jobs"); Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.Count == 2, "There should be two nodes registered"); Assert.IsFalse(checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job queue should be empty."); Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "Job should not be empty."); Assert.AreEqual(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.Count, checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Select(j => j.SentToWorkerNodeUri).Distinct().Count()); checkTablesInManagerDbTimer.Dispose(); var endedTest = DateTime.UtcNow; var description = string.Format("Creates {0} Test Timer jobs with {1} manager and {2} nodes.", numberOfJobs, NumberOfManagers, NumberOfNodes + 1); DatabaseHelper.AddPerformanceData(ManagerDbConnectionString, description, startedTest, endedTest); }
public void ShouldConsiderNodeAsDeadWhenInactiveAndSetJobResultToFatal() { var startedTest = DateTime.UtcNow; var waitForJobToStartEvent = new ManualResetEventSlim(); var waitForNodeToEndEvent = new ManualResetEventSlim(); var checkTablesInManagerDbTimer = new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100); checkTablesInManagerDbTimer.GetJobItems += (sender, items) => { if (items.Any() && items.All(job => job.Started != null)) { waitForJobToStartEvent.Set(); } }; checkTablesInManagerDbTimer.GetWorkerNodes += (sender, nodes) => { if (nodes.Any() && nodes.All(node => node.Alive == false)) { waitForNodeToEndEvent.Set(); } }; checkTablesInManagerDbTimer.JobTimer.Start(); checkTablesInManagerDbTimer.WorkerNodeTimer.Start(); var jobQueueItem = JobHelper.GenerateTestJobRequests(1, 5 * 60).First(); var jobId = HttpRequestManager.AddJob(jobQueueItem); waitForJobToStartEvent.Wait(TimeSpan.FromMinutes(2)); Task <string> taskShutDownNode = new Task <string>(() => { string res = IntegrationControllerApiHelper.ShutDownNode(HttpSender, "Node1.config").Result; return(res); }); taskShutDownNode.Start(); taskShutDownNode.Wait(); waitForNodeToEndEvent.Wait(TimeSpan.FromMinutes(1)); //Give manager a couple of seconds to requeue job Thread.Sleep(TimeSpan.FromSeconds(2)); var jobs = checkTablesInManagerDbTimer.ManagerDbRepository.Jobs; Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.All(node => node.Alive == false), "Worker Node should not be alive."); Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job should be in queue."); Assert.IsTrue(!jobs.Any(), "Job should be empty."); checkTablesInManagerDbTimer.Dispose(); var endedTest = DateTime.UtcNow; var description = string.Format("Creates Node Failure jobs with {0} manager and {1} nodes.", NumberOfManagers, NumberOfNodes); DatabaseHelper.AddPerformanceData(ManagerDbConnectionString, description, startedTest, endedTest); }