public void CreateRequestShouldReturnCancelStatusWhenJobHasStartedAndBeenCanceled()
{
WaitForNodeToFinishWorking();
var startedTest = DateTime.UtcNow;
var manualResetEventSlim = new ManualResetEventSlim();
var checkTablesInManagerDbTimer = new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100);
bool sentCancel = false;
checkTablesInManagerDbTimer.GetJobItems += (sender, items) =>
{
if (items.Any() &&
items.All(job => job.Ended == null) && sentCancel == false)
{
sentCancel = true;
HttpRequestManager.CancelJob(items.First().JobId);
}
if (items.Any() &&
items.All(job => job.Ended != null))
{
manualResetEventSlim.Set();
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
var jobQueueItem =
JobHelper.GenerateTestJobRequests(1, 60).First();
var jobId = HttpRequestManager.AddJob(jobQueueItem);
manualResetEventSlim.Wait(TimeSpan.FromSeconds(10));
Assert.IsTrue(!checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job queue must be empty.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "Jobs must have been added.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(job => job.Result.StartsWith("Canceled", StringComparison.InvariantCultureIgnoreCase)));
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates Cancel Job with {0} manager and {1} nodes.",
NumberOfManagers,
NumberOfNodes);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
public void JobShouldHaveStatusFailedIfFailedTest()
{
WaitForNodeToFinishWorking();
var startedTest = DateTime.UtcNow;
var manualResetEventSlim = new ManualResetEventSlim();
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100);
checkTablesInManagerDbTimer.GetJobItems += (sender, items) =>
{
if (items.Any() &&
items.All(job => job.Ended != null))
{
if (!manualResetEventSlim.IsSet)
{
manualResetEventSlim.Set();
}
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
var failingJobQueueItem =
JobHelper.GenerateFailingJobParamsRequests(1).First();
var jobId = HttpRequestManager.AddJob(failingJobQueueItem);
manualResetEventSlim.Wait(TimeSpan.FromSeconds(10));
Assert.IsTrue(!checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job queue must be empty.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "Jobs must have been added.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.
Jobs.Any(job => job.Result.StartsWith("fail", StringComparison.InvariantCultureIgnoreCase)));
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates FAILED jobs with {0} manager and {1} nodes.",
NumberOfManagers,
NumberOfNodes);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
checkTablesInManagerDbTimer.Dispose();
}
public void ShouldBeAbleToCreateManySuccessJobRequestTest()
{
var startedTest = DateTime.UtcNow;
var manualResetEventSlim = new ManualResetEventSlim();
var createNewJobRequests =
JobHelper.GenerateTestJobRequests(50, 1);
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 2000);
checkTablesInManagerDbTimer.GetJobItems += (sender, jobs) =>
{
if (jobs.Count == createNewJobRequests.Count &&
jobs.All(job => job.Started != null && job.Ended != null))
{
manualResetEventSlim.Set();
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
foreach (var newJobRequest in createNewJobRequests)
{
var jobId = HttpRequestManager.AddJob(newJobRequest);
}
manualResetEventSlim.Wait(TimeSpan.FromMinutes(5));
var jobsToAssert = checkTablesInManagerDbTimer.ManagerDbRepository.Jobs;
Assert.IsTrue(jobsToAssert.Count == createNewJobRequests.Count);
Assert.IsTrue(jobsToAssert.All(job => job.Result.StartsWith("Success", StringComparison.InvariantCultureIgnoreCase)));
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates {0} FAST = 1 second jobs with {1} manager and {2} nodes.",
createNewJobRequests.Count,
NumberOfManagers,
NumberOfNodes);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
public void JobsShouldJustBeQueuedIfNoNodesTest()
{
var startedTest = DateTime.UtcNow;
var manualResetEventSlim = new ManualResetEventSlim();
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100);
checkTablesInManagerDbTimer.GetJobQueueItems += (sender, items) =>
{
if (items.Any())
{
manualResetEventSlim.Set();
}
};
checkTablesInManagerDbTimer.JobQueueTimer.Start();
var jobQueueItem =
JobHelper.GenerateTestJobRequests(1, 5).First();
HttpRequestManager.AddJob(jobQueueItem);
manualResetEventSlim.Wait(TimeSpan.FromSeconds(30));
//Wait a little time to make sure it is only queued and not sent to node
Thread.Sleep(TimeSpan.FromSeconds(5));
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any());
Assert.IsTrue(!checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any());
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates {0} Test Timer jobs with {1} manager and {2} nodes.",
1,
NumberOfManagers,
NumberOfNodes);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
public void ShouldBeAbleToCreateASuccessJobRequestTest()
{
WaitForNodeToFinishWorking();
var startedTest = DateTime.UtcNow;
var manualResetEventSlim = new ManualResetEventSlim();
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 2000);
checkTablesInManagerDbTimer.GetJobItems += (sender, items) =>
{
if (items.Any() &&
items.All(job => job.Started != null && job.Ended != null))
{
manualResetEventSlim.Set();
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
var jobQueueItem =
JobHelper.GenerateTestJobRequests(1, 5).First();
var jobId = HttpRequestManager.AddJob(jobQueueItem);
manualResetEventSlim.Wait(TimeSpan.FromSeconds(30));
Assert.IsTrue(!checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Should not be any jobs left in queue.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "There should be jobs in jobs table.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.
Jobs.All(job => job.Result.StartsWith("success", StringComparison.InvariantCultureIgnoreCase)));
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates {0} Test Timer jobs with {1} manager and {2} nodes.",
1,
NumberOfManagers,
NumberOfNodes);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
public void NodeShouldNotGetStuckWhenCancellingOrFailingJobs()
{
WaitForNodeToFinishWorking();
var jobRepository = new ManagerDbRepository(ManagerDbConnectionString);
var jobQueueItemCancel = JobHelper.GenerateTestJobRequests(1, 60).First();
var jobQueueItemFail = JobHelper.GenerateFailingJobParamsRequests(1).First();
var jobQueueItemSuccess = JobHelper.GenerateTestJobRequests(1, 2).First();
var cancelId = HttpRequestManager.AddJob(jobQueueItemCancel);
WaitForNodeToStartWorking();
HttpRequestManager.CancelJob(cancelId);
WaitForNodeToFinishWorking();
Thread.Sleep(TimeSpan.FromSeconds(2));
HttpRequestManager.AddJob(jobQueueItemFail);
Thread.Sleep(TimeSpan.FromSeconds(2)); //might be a risc it ends before we see it is working..
WaitForNodeToFinishWorking();
Thread.Sleep(TimeSpan.FromSeconds(2));
HttpRequestManager.AddJob(jobQueueItemSuccess);
WaitForNodeToStartWorking();
WaitForNodeToFinishWorking();
Thread.Sleep(TimeSpan.FromSeconds(2));
var queueItems = jobRepository.JobQueueItems;
var jobs = jobRepository.Jobs;
Assert.IsTrue(!queueItems.Any(), "Job queue must be empty.");
Assert.IsTrue(jobs.Any(), "Jobs must have been added.");
Assert.IsTrue(jobs.Any(job => job.Result.StartsWith("Canceled", StringComparison.InvariantCultureIgnoreCase)));
Assert.IsTrue(jobs.Any(job => job.Result.StartsWith("fail", StringComparison.InvariantCultureIgnoreCase)));
Assert.IsTrue(jobs.Any(job => job.Result.StartsWith("success", StringComparison.InvariantCultureIgnoreCase)));
}
public void ShouldHandleNodeDisapperingTemporarily()
{
var startedTest = DateTime.UtcNow;
var numberOfJobs = 20;
var waitForJobToFinishEvent = new ManualResetEventSlim();
var waitForNodeToStartEvent = new ManualResetEventSlim();
var waitForAllNodesToFinishAJob = new ManualResetEventSlim();
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 300);
checkTablesInManagerDbTimer.GetJobItems += (sender, items) =>
{
if (items.Count(job => job.Ended != null) == numberOfJobs)
{
waitForJobToFinishEvent.Set();
}
if (items.Where(job => job.Ended != null).GroupBy(job => job.SentToWorkerNodeUri).Count() == 2)
{
waitForAllNodesToFinishAJob.Set();
}
};
checkTablesInManagerDbTimer.GetWorkerNodes += (sender, nodes) =>
{
if (nodes.Count == 2)
{
waitForNodeToStartEvent.Set();
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
checkTablesInManagerDbTimer.WorkerNodeTimer.Start();
//start second node
Task <string> taskStartNewNode = new Task <string>(() =>
{
string res = IntegrationControllerApiHelper.StartNewNode(HttpSender).Result;
return(res);
});
taskStartNewNode.Start();
waitForNodeToStartEvent.Wait();
var jobQueueItemsBatch1 = JobHelper.GenerateTestJobRequests(numberOfJobs, 1);
jobQueueItemsBatch1.ForEach(jobQueueItem => HttpRequestManager.AddJob(jobQueueItem));
var allNodesRunningWithoutTimeout = waitForAllNodesToFinishAJob.Wait(TimeSpan.FromSeconds(60));
Assert.IsTrue(allNodesRunningWithoutTimeout, "Timeout when waiting for all nodes to finish at least one job");
Assert.AreNotEqual(0, checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueCount);
Task <string> taskShutDownNode2 = new Task <string>(() =>
{
string res = IntegrationControllerApiHelper.ShutDownNode(HttpSender, "Node2.config").Result;
return(res);
});
taskShutDownNode2.RunSynchronously();
//restart second node
Task <string> taskStartNewNodeRestarted = new Task <string>(() =>
{
string res = IntegrationControllerApiHelper.StartNewNode(HttpSender).Result;
return(res);
});
taskStartNewNodeRestarted.RunSynchronously();
var jobsFinishedWithoutTimeout = waitForJobToFinishEvent.Wait(TimeSpan.FromSeconds(200));
Assert.IsTrue(jobsFinishedWithoutTimeout, "Timeout on Finishing jobs");
//
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.Count == 3, "There should be two nodes registered");
Assert.IsFalse(checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job queue should be empty.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "Job should not be empty.");
Assert.AreEqual(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.Count,
checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Select(j => j.SentToWorkerNodeUri).Distinct().Count());
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates {0} Test Timer jobs with {1} manager and {2} nodes.",
numberOfJobs,
NumberOfManagers,
NumberOfNodes + 2);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
public void ShouldHandleMultipleJobsUsingAllNodesAvailable()
{
var startedTest = DateTime.UtcNow;
var numberOfJobs = 15;
var waitForJobToFinishEvent = new ManualResetEventSlim();
var waitForNodeToStartEvent = new ManualResetEventSlim();
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100);
checkTablesInManagerDbTimer.GetJobItems += (sender, items) =>
{
if (items.Count(j => j.Ended != null) == numberOfJobs)
{
waitForJobToFinishEvent.Set();
}
};
checkTablesInManagerDbTimer.GetWorkerNodes += (sender, nodes) =>
{
if (nodes.Count == 2)
{
waitForNodeToStartEvent.Set();
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
checkTablesInManagerDbTimer.WorkerNodeTimer.Start();
//start second node
Task <string> taskStartNewNode = new Task <string>(() =>
{
string res = IntegrationControllerApiHelper.StartNewNode(HttpSender).Result;
return(res);
});
taskStartNewNode.Start();
waitForNodeToStartEvent.Wait();
var jobQueueItems = JobHelper.GenerateTestJobRequests(numberOfJobs, 1);
jobQueueItems.ForEach(jobQueueItem => HttpRequestManager.AddJob(jobQueueItem));
var jobsFinishedWithoutTimeout = waitForJobToFinishEvent.Wait(TimeSpan.FromSeconds(60));
Assert.IsTrue(jobsFinishedWithoutTimeout, "Timeout on Finishing jobs");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.Count == 2, "There should be two nodes registered");
Assert.IsFalse(checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job queue should be empty.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "Job should not be empty.");
Assert.AreEqual(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.Count,
checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Select(j => j.SentToWorkerNodeUri).Distinct().Count());
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates {0} Test Timer jobs with {1} manager and {2} nodes.",
numberOfJobs,
NumberOfManagers,
NumberOfNodes + 1);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
public void ShouldConsiderNodeAsDeadWhenInactiveAndSetJobResultToFatal()
{
var startedTest = DateTime.UtcNow;
var waitForJobToStartEvent = new ManualResetEventSlim();
var waitForNodeToEndEvent = new ManualResetEventSlim();
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100);
checkTablesInManagerDbTimer.GetJobItems += (sender, items) =>
{
if (items.Any() && items.All(job => job.Started != null))
{
waitForJobToStartEvent.Set();
}
};
checkTablesInManagerDbTimer.GetWorkerNodes += (sender, nodes) =>
{
if (nodes.Any() && nodes.All(node => node.Alive == false))
{
waitForNodeToEndEvent.Set();
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
checkTablesInManagerDbTimer.WorkerNodeTimer.Start();
var jobQueueItem =
JobHelper.GenerateTestJobRequests(1, 5 * 60).First();
var jobId = HttpRequestManager.AddJob(jobQueueItem);
waitForJobToStartEvent.Wait(TimeSpan.FromMinutes(2));
Task <string> taskShutDownNode = new Task <string>(() =>
{
string res = IntegrationControllerApiHelper.ShutDownNode(HttpSender,
"Node1.config").Result;
return(res);
});
taskShutDownNode.Start();
taskShutDownNode.Wait();
waitForNodeToEndEvent.Wait(TimeSpan.FromMinutes(1));
//Give manager a couple of seconds to requeue job
Thread.Sleep(TimeSpan.FromSeconds(2));
var jobs = checkTablesInManagerDbTimer.ManagerDbRepository.Jobs;
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.All(node => node.Alive == false), "Worker Node should not be alive.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job should be in queue.");
Assert.IsTrue(!jobs.Any(), "Job should be empty.");
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates Node Failure jobs with {0} manager and {1} nodes.",
NumberOfManagers,
NumberOfNodes);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}