public void SemaphoreTest()
{
Assert.That(SemaphoreRepository.MaxAge > TimeSpan.FromSeconds(2),
"Can not complete the test with a SemaphoreMaxAge less than 2 sec.");
var callerId1 = "CommonRepoTest1";
string currentOwner;
Assert.That(SemaphoreRepository.Get(_semaphoreId, callerId1, out currentOwner), Is.True,
"Should get lock from empty repo");
Assert.That(callerId1, Is.EqualTo(currentOwner),
"Current owner should be set correctly");
// Make a time step less than SemaphoreMaxAge
TimeProvider.Step(SemaphoreRepository.MaxAge.Add(TimeSpan.FromSeconds(-1)));
var callerId2 = "CommonRepoTest2";
Assert.That(SemaphoreRepository.Get(_semaphoreId, callerId2, out currentOwner), Is.False,
"Another caller should not get the lock before max age has expired.");
Assert.That(callerId1, Is.EqualTo(currentOwner),
"Current owner should not have changed.");
Assert.That(SemaphoreRepository.Get(_semaphoreId, callerId1, out currentOwner), Is.True,
"First caller should still get the lock");
Assert.That(callerId1, Is.EqualTo(currentOwner),
"Current owner should not have changed.");
// Make a time step more than SemaphoreMaxAge
TimeProvider.Step(SemaphoreRepository.MaxAge.Add(TimeSpan.FromSeconds(1)));
Assert.That(SemaphoreRepository.Get(_semaphoreId, callerId2, out currentOwner), Is.True,
"The semaphore max age has been exceded ownership change should be allowed.");
Assert.That(callerId2, Is.EqualTo(currentOwner),
"The current owner is changed correctly to 2nd caller id.");
Assert.That(SemaphoreRepository.Get(_semaphoreId, callerId1, out currentOwner), Is.False,
"The previous caller is denied, since the semaphore has moved to 2nd caller");
Assert.That(callerId2, Is.EqualTo(currentOwner),
"Current owner should not have changed.");
var probe = SemaphoreRepository.Probe(_semaphoreId);
Assert.That(probe.SemaphoreId, Is.EqualTo(_semaphoreId));
Assert.That(probe.CurrentOwnerId, Is.EqualTo(callerId2));
Assert.That(probe.HeartBeat, Is.EqualTo(TimeProvider.GetUtcNow()));
}
public void Pulse()
{
lock (_jobRepository)
{
#region semaphore
string currentOwner;
var firstCallAfterSemaphore = _firstPulse || !_wasMaster;
if (!_semaphoreRepository.Get(nameof(Executor), Utilities.GetCallerId(), out currentOwner))
{
if (_firstPulse)
{
_logging.LogInfo($"Did not get semaphore, current owner : {currentOwner}. Will stand by as slave.");
_firstPulse = false;
}
else if (_wasMaster)
{
_logging.LogWarning($"Lost semaphore to : {currentOwner}. Will stand by as slave.");
_wasMaster = false;
}
return;
}
if (_firstPulse)
{
_logging.LogInfo($"Got semaphore, as : {currentOwner}. Will work as master.");
_firstPulse = false;
}
else if (!_wasMaster)
{
_logging.LogInfo($"Slave woken, {currentOwner} is now owner. Will work as master.");
foreach (var plugin in _plugins)
{
plugin.Reset();
}
}
_wasMaster = true;
#endregion
#region command
foreach (var command in _commandRepository.GetAll())
{
// ReSharper disable once SwitchStatementMissingSomeCases
switch (command.Type)
{
case CommandType.Cancel:
CancelJob(command.Urn, command.Username);
break;
default:
_logging.LogWarning($"Command state {command.Type} is not implemented.", command.Urn);
break;
}
_commandRepository.Remove(command);
}
#endregion
#region planner
// TODO: modify existing plans ? :hamburger: :+1:
if (!firstCallAfterSemaphore) // skip first pulse to reassign in-progress tasks to plugins.
{
_planner.Calculate();
}
#endregion
#region jobs, task and plugins
foreach (var job in _jobRepository.ActiveJobs().ToList())
{
//TODO: Add support for cancel
var plan = job.Plan;
startOfJobLoop:
var executionTask = plan.GetCurrentTask();
var targetPlugin = _plugins.First(p => p.Urn == executionTask.PluginUrn);
switch (executionTask.State)
{
case ExecutionState.Queued:
if (targetPlugin.Busy)
{
// TODO: log planning warning
break;
}
_logging.LogDebug($"Task {executionTask.Urn} assigned to {targetPlugin.Urn}.", job.Urn);
targetPlugin.Assign(executionTask);
goto case ExecutionState.Running;
case ExecutionState.Running:
targetPlugin.Pulse(executionTask);
if (executionTask.State == ExecutionState.Done)
{
goto case ExecutionState.Done;
}
if (executionTask.State == ExecutionState.Failed)
{
goto case ExecutionState.Failed;
}
plan.Tasks[plan.ActiveTaskIndex.Value] = executionTask;
break;
case ExecutionState.Done:
_logging.LogDebug($"Task {executionTask.Urn} done, released from {targetPlugin.Urn}.", job.Urn);
targetPlugin.Release(executionTask);
plan.Tasks[plan.ActiveTaskIndex.Value] = executionTask;
plan.MoveToNextTask();
if (plan.ActiveTaskIndex.HasValue) // has more tasks
{
_jobRepository.Update(job); // save and...
goto startOfJobLoop; //start next task at once
}
break;
case ExecutionState.Failed:
if (targetPlugin.CanRetry && executionTask.NumberOfRetries < targetPlugin.RetryMax)
{
targetPlugin.Retry(executionTask);
}
break;
default:
throw new ArgumentOutOfRangeException();
}
if (plan.GetState() == ExecutionState.Done)
{
job.Destination = plan.GetCurrentEssence();
job.EndTime = _timeProvider.GetUtcNow();
_logging.LogInfo("Job done", job.Urn);
}
if (plan.GetState() == ExecutionState.Failed)
{
_logging.LogWarning("Job failed", job.Urn);
}
if ((plan.GetState() == ExecutionState.Failed || plan.GetState() == ExecutionState.Done) && !string.IsNullOrEmpty(job.CallbackUrl))
{
MakeCallback(job);
}
_jobRepository.Update(job);
}
}
#endregion
}
