public void Execute(object parameter)
{
Task.Factory.StartNew(() =>
{
var httpSender = new HttpSender();
var managerName = IntegrationControllerApiHelper.StartNewManager(httpSender);
});
}
public void ShouldHandleMultipleJobsUsingAllNodesAvailable()
{
var startedTest = DateTime.UtcNow;
var numberOfJobs = 15;
var waitForJobToFinishEvent = new ManualResetEventSlim();
var waitForNodeToStartEvent = new ManualResetEventSlim();
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100);
checkTablesInManagerDbTimer.GetJobItems += (sender, items) =>
{
if (items.Count(j => j.Ended != null) == numberOfJobs)
{
waitForJobToFinishEvent.Set();
}
};
checkTablesInManagerDbTimer.GetWorkerNodes += (sender, nodes) =>
{
if (nodes.Count == 2)
{
waitForNodeToStartEvent.Set();
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
checkTablesInManagerDbTimer.WorkerNodeTimer.Start();
//start second node
Task <string> taskStartNewNode = new Task <string>(() =>
{
string res = IntegrationControllerApiHelper.StartNewNode(HttpSender).Result;
return(res);
});
taskStartNewNode.Start();
waitForNodeToStartEvent.Wait();
var jobQueueItems = JobHelper.GenerateTestJobRequests(numberOfJobs, 1);
jobQueueItems.ForEach(jobQueueItem => HttpRequestManager.AddJob(jobQueueItem));
var jobsFinishedWithoutTimeout = waitForJobToFinishEvent.Wait(TimeSpan.FromSeconds(60));
Assert.IsTrue(jobsFinishedWithoutTimeout, "Timeout on Finishing jobs");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.Count == 2, "There should be two nodes registered");
Assert.IsFalse(checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job queue should be empty.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "Job should not be empty.");
Assert.AreEqual(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.Count,
checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Select(j => j.SentToWorkerNodeUri).Distinct().Count());
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates {0} Test Timer jobs with {1} manager and {2} nodes.",
numberOfJobs,
NumberOfManagers,
NumberOfNodes + 1);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
public void ShouldHandleNodeDisapperingTemporarily()
{
var startedTest = DateTime.UtcNow;
var numberOfJobs = 20;
var waitForJobToFinishEvent = new ManualResetEventSlim();
var waitForNodeToStartEvent = new ManualResetEventSlim();
var waitForAllNodesToFinishAJob = new ManualResetEventSlim();
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 300);
checkTablesInManagerDbTimer.GetJobItems += (sender, items) =>
{
if (items.Count(job => job.Ended != null) == numberOfJobs)
{
waitForJobToFinishEvent.Set();
}
if (items.Where(job => job.Ended != null).GroupBy(job => job.SentToWorkerNodeUri).Count() == 2)
{
waitForAllNodesToFinishAJob.Set();
}
};
checkTablesInManagerDbTimer.GetWorkerNodes += (sender, nodes) =>
{
if (nodes.Count == 2)
{
waitForNodeToStartEvent.Set();
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
checkTablesInManagerDbTimer.WorkerNodeTimer.Start();
//start second node
Task <string> taskStartNewNode = new Task <string>(() =>
{
string res = IntegrationControllerApiHelper.StartNewNode(HttpSender).Result;
return(res);
});
taskStartNewNode.Start();
waitForNodeToStartEvent.Wait();
var jobQueueItemsBatch1 = JobHelper.GenerateTestJobRequests(numberOfJobs, 1);
jobQueueItemsBatch1.ForEach(jobQueueItem => HttpRequestManager.AddJob(jobQueueItem));
var allNodesRunningWithoutTimeout = waitForAllNodesToFinishAJob.Wait(TimeSpan.FromSeconds(60));
Assert.IsTrue(allNodesRunningWithoutTimeout, "Timeout when waiting for all nodes to finish at least one job");
Assert.AreNotEqual(0, checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueCount);
Task <string> taskShutDownNode2 = new Task <string>(() =>
{
string res = IntegrationControllerApiHelper.ShutDownNode(HttpSender, "Node2.config").Result;
return(res);
});
taskShutDownNode2.RunSynchronously();
//restart second node
Task <string> taskStartNewNodeRestarted = new Task <string>(() =>
{
string res = IntegrationControllerApiHelper.StartNewNode(HttpSender).Result;
return(res);
});
taskStartNewNodeRestarted.RunSynchronously();
var jobsFinishedWithoutTimeout = waitForJobToFinishEvent.Wait(TimeSpan.FromSeconds(200));
Assert.IsTrue(jobsFinishedWithoutTimeout, "Timeout on Finishing jobs");
//
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.Count == 3, "There should be two nodes registered");
Assert.IsFalse(checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job queue should be empty.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "Job should not be empty.");
Assert.AreEqual(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.Count,
checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Select(j => j.SentToWorkerNodeUri).Distinct().Count());
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates {0} Test Timer jobs with {1} manager and {2} nodes.",
numberOfJobs,
NumberOfManagers,
NumberOfNodes + 2);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
public void ShouldConsiderNodeAsDeadWhenInactiveAndSetJobResultToFatal()
{
var startedTest = DateTime.UtcNow;
var waitForJobToStartEvent = new ManualResetEventSlim();
var waitForNodeToEndEvent = new ManualResetEventSlim();
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100);
checkTablesInManagerDbTimer.GetJobItems += (sender, items) =>
{
if (items.Any() && items.All(job => job.Started != null))
{
waitForJobToStartEvent.Set();
}
};
checkTablesInManagerDbTimer.GetWorkerNodes += (sender, nodes) =>
{
if (nodes.Any() && nodes.All(node => node.Alive == false))
{
waitForNodeToEndEvent.Set();
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
checkTablesInManagerDbTimer.WorkerNodeTimer.Start();
var jobQueueItem =
JobHelper.GenerateTestJobRequests(1, 5 * 60).First();
var jobId = HttpRequestManager.AddJob(jobQueueItem);
waitForJobToStartEvent.Wait(TimeSpan.FromMinutes(2));
Task <string> taskShutDownNode = new Task <string>(() =>
{
string res = IntegrationControllerApiHelper.ShutDownNode(HttpSender,
"Node1.config").Result;
return(res);
});
taskShutDownNode.Start();
taskShutDownNode.Wait();
waitForNodeToEndEvent.Wait(TimeSpan.FromMinutes(1));
//Give manager a couple of seconds to requeue job
Thread.Sleep(TimeSpan.FromSeconds(2));
var jobs = checkTablesInManagerDbTimer.ManagerDbRepository.Jobs;
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.All(node => node.Alive == false), "Worker Node should not be alive.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job should be in queue.");
Assert.IsTrue(!jobs.Any(), "Job should be empty.");
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates Node Failure jobs with {0} manager and {1} nodes.",
NumberOfManagers,
NumberOfNodes);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
