public void CreateRequestShouldReturnCancelStatusWhenJobHasStartedAndBeenCanceled()
{
var startedTest = DateTime.UtcNow;
var manualResetEventSlim = new ManualResetEventSlim();
var checkTablesInManagerDbTimer = new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100);
bool sentCancel = false;
checkTablesInManagerDbTimer.GetJobItems += (sender, items) =>
{
if (items.Any() &&
items.All(job => job.Ended == null) && sentCancel == false)
{
sentCancel = true;
HttpRequestManager.CancelJob(items.First().JobId);
}
if (items.Any() &&
items.All(job => job.Ended != null))
{
manualResetEventSlim.Set();
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
var jobQueueItem =
JobHelper.GenerateTestJobRequests(1, TimeSpan.FromMinutes(1)).First();
var jobId = HttpRequestManager.AddJob(jobQueueItem);
manualResetEventSlim.Wait(TimeSpan.FromSeconds(30));
Assert.IsTrue(!checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job queue must be empty.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "Jobs must have been added.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(job => job.Result.StartsWith("Canceled", StringComparison.InvariantCultureIgnoreCase)));
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates Cancel Job with {0} manager and {1} nodes.",
NumberOfManagers,
NumberOfNodes);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
public void ShouldBeAbleToCreateManySuccessJobRequestTest()
{
var startedTest = DateTime.UtcNow;
var manualResetEventSlim = new ManualResetEventSlim();
var createNewJobRequests =
JobHelper.GenerateTestJobRequests(50, TimeSpan.FromSeconds(1));
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 2000);
checkTablesInManagerDbTimer.GetJobItems += (sender, jobs) =>
{
if (jobs.Count == createNewJobRequests.Count &&
jobs.All(job => job.Started != null && job.Ended != null))
{
manualResetEventSlim.Set();
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
foreach (var newJobRequest in createNewJobRequests)
{
var jobId = HttpRequestManager.AddJob(newJobRequest);
}
manualResetEventSlim.Wait(TimeSpan.FromMinutes(5));
var jobsToAssert = checkTablesInManagerDbTimer.ManagerDbRepository.Jobs;
Assert.IsTrue(jobsToAssert.Count == createNewJobRequests.Count);
Assert.IsTrue(jobsToAssert.All(job => job.Result.StartsWith("Success", StringComparison.InvariantCultureIgnoreCase)));
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates {0} FAST = 1 second jobs with {1} manager and {2} nodes.",
createNewJobRequests.Count,
NumberOfManagers,
NumberOfNodes);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
public void JobsShouldJustBeQueuedIfNoNodesTest()
{
var startedTest = DateTime.UtcNow;
var manualResetEventSlim = new ManualResetEventSlim();
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100);
checkTablesInManagerDbTimer.GetJobQueueItems += (sender, items) =>
{
if (items.Any())
{
manualResetEventSlim.Set();
}
};
checkTablesInManagerDbTimer.JobQueueTimer.Start();
var jobQueueItem =
JobHelper.GenerateTestJobRequests(1, TimeSpan.FromSeconds(5)).First();
HttpRequestManager.AddJob(jobQueueItem);
manualResetEventSlim.Wait(TimeSpan.FromSeconds(30));
//Wait a little time to make sure it is only queued and not sent to node
Thread.Sleep(TimeSpan.FromSeconds(5));
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any());
Assert.IsTrue(!checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any());
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates {0} Test Timer jobs with {1} manager and {2} nodes.",
1,
NumberOfManagers,
NumberOfNodes);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
public void ShouldConsiderNodeAsDeadWhenInactiveAndSetJobResultToFatal()
{
var startedTest = DateTime.UtcNow;
var waitForJobToStartEvent = new ManualResetEventSlim();
var waitForNodeToEndEvent = new ManualResetEventSlim();
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100);
checkTablesInManagerDbTimer.GetJobItems += (sender, items) =>
{
if (items.Any() && items.All(job => job.Started != null))
{
waitForJobToStartEvent.Set();
}
};
checkTablesInManagerDbTimer.GetWorkerNodes += (sender, nodes) =>
{
if (nodes.Any() && nodes.All(node => node.Alive == false))
{
waitForNodeToEndEvent.Set();
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
checkTablesInManagerDbTimer.WorkerNodeTimer.Start();
var jobQueueItem =
JobHelper.GenerateTestJobRequests(1, TimeSpan.FromMinutes(5)).First();
var jobId = HttpRequestManager.AddJob(jobQueueItem);
waitForJobToStartEvent.Wait(TimeSpan.FromMinutes(2));
Task<string> taskShutDownNode = new Task<string>(() =>
{
string res = IntegrationControllerApiHelper.ShutDownNode(HttpSender,
"Node1.config").Result;
return res;
});
taskShutDownNode.Start();
taskShutDownNode.Wait();
waitForNodeToEndEvent.Wait(TimeSpan.FromMinutes(1));
//Give manager a couple of seconds to set the result to fatal
Thread.Sleep(TimeSpan.FromSeconds(2));
var jobs = checkTablesInManagerDbTimer.ManagerDbRepository.Jobs;
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.All(node => node.Alive == false), "Worker Node should not be alive.");
Assert.IsTrue(!checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(),"Job queue should be empty.");
Assert.IsTrue(jobs.Any(), "Job should not be empty.");
Assert.IsTrue(jobs.Any(job => job.Result.StartsWith("Fatal Node Failure",StringComparison.InvariantCultureIgnoreCase)),
"All jobs shall have status of Fatal Node Failure.");
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates Node Failure jobs with {0} manager and {1} nodes.",
NumberOfManagers,
NumberOfNodes);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
public void ShouldSendAJobToTheNextNodeIfTheFirstIsNotResponding()
{
var startedTest = DateTime.UtcNow;
var waitForJobToStartEvent = new ManualResetEventSlim();
var waitForNodeToStartEvent = new ManualResetEventSlim();
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100);
checkTablesInManagerDbTimer.GetJobItems += (sender, items) =>
{
if (items.Any() && items.All(job => job.Started != null))
{
waitForJobToStartEvent.Set();
}
};
checkTablesInManagerDbTimer.GetWorkerNodes += (sender, nodes) =>
{
if (nodes.Count == 2)
{
waitForNodeToStartEvent.Set();
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
checkTablesInManagerDbTimer.WorkerNodeTimer.Start();
//start second node
Task<string> taskStartNewNode = new Task<string>(() =>
{
string res = IntegrationControllerApiHelper.StartNewNode(HttpSender).Result;
return res;
});
taskStartNewNode.Start();
waitForNodeToStartEvent.Wait();
//kill second node
Task<string> taskShutDownNode = new Task<string>(() =>
{
string res = IntegrationControllerApiHelper.ShutDownNode(HttpSender,"Node2.config").Result;
return res;
});
taskShutDownNode.Start();
var jobQueueItem = JobHelper.GenerateTestJobRequests(1, TimeSpan.FromSeconds(1)).First();
var jobId = HttpRequestManager.AddJob(jobQueueItem);
//Should not take long time to assign job, if it takes more than some seconds
// otherwise it is maybe the background timer who assigned the job
waitForJobToStartEvent.Wait(TimeSpan.FromSeconds(10));
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.WorkerNodes.Count == 2, "There should be two nodes registered");
Assert.IsTrue(!checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job queue should be empty.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "Job should not be empty.");
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates {0} Test Timer jobs with {1} manager and {2} nodes.",
1,
NumberOfManagers,
NumberOfNodes);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
public void ShouldBeAbleToCreateASuccessJobRequestTest()
{
var startedTest = DateTime.UtcNow;
var manualResetEventSlim = new ManualResetEventSlim();
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 2000);
checkTablesInManagerDbTimer.GetJobItems += (sender, items) =>
{
if (items.Any() &&
items.All(job => job.Started != null && job.Ended != null))
{
manualResetEventSlim.Set();
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
var jobQueueItem =
JobHelper.GenerateTestJobRequests(1, TimeSpan.FromSeconds(5)).First();
var jobId = HttpRequestManager.AddJob(jobQueueItem);
manualResetEventSlim.Wait(TimeSpan.FromSeconds(30));
Assert.IsTrue(!checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Should not be any jobs left in queue.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "There should be jobs in jobs table.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.
Jobs.All(job => job.Result.StartsWith("success", StringComparison.InvariantCultureIgnoreCase)));
checkTablesInManagerDbTimer.Dispose();
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates {0} Test Timer jobs with {1} manager and {2} nodes.",
1,
NumberOfManagers,
NumberOfNodes);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
}
public void JobShouldHaveStatusFailedIfFailedTest()
{
var startedTest = DateTime.UtcNow;
var manualResetEventSlim = new ManualResetEventSlim();
var checkTablesInManagerDbTimer =
new CheckTablesInManagerDbTimer(ManagerDbConnectionString, 100);
checkTablesInManagerDbTimer.GetJobItems += (sender, items) =>
{
if (items.Any() &&
items.All(job => job.Ended != null))
{
if (!manualResetEventSlim.IsSet)
{
manualResetEventSlim.Set();
}
}
};
checkTablesInManagerDbTimer.JobTimer.Start();
var failingJobQueueItem =
JobHelper.GenerateFailingJobParamsRequests(1).First();
var jobId = HttpRequestManager.AddJob(failingJobQueueItem);
manualResetEventSlim.Wait(TimeSpan.FromSeconds(10));
Assert.IsTrue(!checkTablesInManagerDbTimer.ManagerDbRepository.JobQueueItems.Any(), "Job queue must be empty.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.Jobs.Any(), "Jobs must have been added.");
Assert.IsTrue(checkTablesInManagerDbTimer.ManagerDbRepository.
Jobs.Any(job => job.Result.StartsWith("fail", StringComparison.InvariantCultureIgnoreCase)));
var endedTest = DateTime.UtcNow;
var description =
string.Format("Creates FAILED jobs with {0} manager and {1} nodes.",
NumberOfManagers,
NumberOfNodes);
DatabaseHelper.AddPerformanceData(ManagerDbConnectionString,
description,
startedTest,
endedTest);
checkTablesInManagerDbTimer.Dispose();
}