/// <summary>
/// Cleanup and synchronize running jobs and jobs table.
/// * Job is deleted based on AutoDeletePeriod and AutoDeleteStatus settings.
/// * Mark job as an error, when job status is "RUNNING" in DB table, but there is no actual running thread in the related server process (Zombie Jobs).
/// * Remove thread references in memory, when job is deleted or status in DB is: stopped, error, or completed.
/// </summary>
public async Task CleanUpAsync(bool isSync)
{
if (isSync)
{
StopJobsAsync(isSync).GetAwaiter().GetResult();
//Delete past completed jobs from storage
if (config.AutoDeletePeriod != null)
{
jobDAL.Delete(config.AutoDeletePeriod.Value, config.AutoDeleteStatus);
}
}
else
{
await StopJobsAsync(isSync);
//Delete past completed jobs from storage
if (config.AutoDeletePeriod != null)
{
await jobDAL.DeleteAsync(config.AutoDeletePeriod.Value, config.AutoDeleteStatus);
}
}
//Get all running process with ProcessID
var jobList = isSync ? jobDAL.GetJobsByProcessAndStatus(workerProcessID, JobStatus.Running) : await jobDAL.GetJobsByProcessAndStatusAsync(workerProcessID, JobStatus.Running);
foreach (var job in jobList)
{
if (!taskList.ContainsKey(job.JobID))
{
//Doesn't exist anymore?
var error = "Error: No actual running job process found. Try reset and run again.";
var processID = string.IsNullOrWhiteSpace(job.ProcessID) ? workerProcessID : job.ProcessID;
var count = isSync ? SetErrorAsync(processID, job.JobID, error, isSync).GetAwaiter().GetResult()
: await SetErrorAsync(processID, job.JobID, error, isSync);
}
}
if (taskList.Count > 0)
{
var inDBjobIDs = new List <string>();
jobList = isSync ? jobDAL.GetJobs(taskList.Keys.ToList()) : await jobDAL.GetJobsAsync(taskList.Keys.ToList()); //get all jobs in taskList
// If jobs doesn't even exists in storage (zombie?), remove from taskList.
inDBjobIDs = jobList.Select(j => j.JobID).ToList();
var taskListKeys = new List <string>(taskList.Keys); //copy keys before removal
foreach (var jobID in taskListKeys)
{
if (!inDBjobIDs.Contains(jobID))
{
TaskInfo taskInfo = null;
if (taskList.Keys.Contains(jobID))
{
taskInfo = taskList[jobID];
}
else
{
continue;
}
taskInfo.TokenSource.Cancel(); //attempt to cancel
taskList.Remove(jobID);
}
}
// For job status that is stopped, error, completed => Remove from thread list, no need to keep track of them anymore.
var statuses = new List <int>
{
(int)JobStatus.Stopped,
(int)JobStatus.Error,
(int)JobStatus.Completed
};
foreach (var job in jobList)
{
if (job.Status != null &&
statuses.Contains((int)job.Status) &&
taskList.ContainsKey(job.JobID))
{
var taskInfo = taskList[job.JobID];
taskInfo.TokenSource.Dispose();
taskList.Remove(job.JobID);
}
}
}
}
/// <summary>
/// Cleanup and synchronize running jobs and jobs table.
/// * Job is deleted based on AutoDeletePeriod and AutoDeleteStatus settings.
/// * Mark job as an error, when job status is "RUNNING" in DB table, but there is no actual running thread in the related server process (Zombie Jobs).
/// * Mark job as an error, when job status is "PAUSED" in DB table, but there is no actual running thread in the related server process (Zombie Jobs), can't run it again.
/// * Remove thread references in memory, when job is deleted or status in DB is: stopped, error, or completed.
/// </summary>
public async Task CleanUpAsync(bool isSync)
{
if (isSync)
{
StopJobsAsync(isSync).GetAwaiter().GetResult();
//Delete past completed jobs from storage
if (this.autoDeletePeriod != null)
{
jobDAL.Delete(this.autoDeletePeriod.Value, this.autoDeleteStatus);
}
}
else
{
await StopJobsAsync(isSync);
//Delete past completed jobs from storage
if (this.autoDeletePeriod != null)
{
await jobDAL.DeleteAsync(this.autoDeletePeriod.Value, this.autoDeleteStatus);
}
}
//Get all RUNNING process from this worker ProcessID
var jobList = isSync ? jobDAL.GetJobsByProcessAndStatus(workerProcessID, JobStatus.Running) : await jobDAL.GetJobsByProcessAndStatusAsync(workerProcessID, JobStatus.Running);
foreach (var job in jobList)
{
if (!taskList.ContainsKey(job.JobID))
{
//Doesn't exist anymore?
var error = "Error: No actual running job process found. Try reset and run again.";
var processID = string.IsNullOrWhiteSpace(job.ProcessID) ? workerProcessID : job.ProcessID;
var count = isSync ? SetErrorAsync(processID, job.JobID, error, isSync).GetAwaiter().GetResult()
: await SetErrorAsync(processID, job.JobID, error, isSync);
}
}
//Get all PAUSED process from this worker ProcessID
jobList = isSync ? jobDAL.GetJobsByProcessAndStatus(workerProcessID, JobStatus.Paused) : await jobDAL.GetJobsByProcessAndStatusAsync(workerProcessID, JobStatus.Paused);
foreach (var job in jobList)
{
if (!taskList.ContainsKey(job.JobID))
{
//Doesn't exist anymore?
var error = "Error: No actual running job process found, unable to continue paused job. Try reset and run again.";
var processID = string.IsNullOrWhiteSpace(job.ProcessID) ? workerProcessID : job.ProcessID;
var count = isSync ? SetErrorAsync(processID, job.JobID, error, isSync).GetAwaiter().GetResult()
: await SetErrorAsync(processID, job.JobID, error, isSync);
}
}
//Synchronize what's in taskList status and in DB
//Remove all non-running jobs from taskList
if (taskList.Count > 0)
{
var inDBjobIDs = new List <string>();
jobList = isSync ? jobDAL.GetJobs(taskList.Keys.ToList()) : await jobDAL.GetJobsAsync(taskList.Keys.ToList()); //get all jobs in taskList
// If jobs doesn't even exists in storage (deleted manually?), remove from taskList.
inDBjobIDs = jobList.Select(j => j.JobID).ToList();
var removalList = new List <string>();
foreach (var jobID in taskList.Keys)
{
//jobID is not in DB?
if (!inDBjobIDs.Contains(jobID))
{
removalList.Add(jobID);
}
}
//Try to Stop/Cancel jobs and remove from taskList
if (isSync)
{
StopJobsAsync(new ReadOnlyCollection <string>(removalList), isSync).GetAwaiter().GetResult();
}
else
{
await StopJobsAsync(new ReadOnlyCollection <string>(removalList), isSync);
}
// For job status that is stopped, error, completed => Remove from thread list, no need to keep track of them anymore.
var statuses = new List <int>
{
(int)JobStatus.Stopped,
(int)JobStatus.Error,
(int)JobStatus.Completed
};
foreach (var job in jobList)
{
if (job.Status != null &&
statuses.Contains((int)job.Status) &&
taskList.ContainsKey(job.JobID))
{
var taskInfo = taskList[job.JobID];
if (taskInfo.CancelSource != null)
{
taskInfo.CancelSource.Dispose();
}
taskList.Remove(job.JobID);
}
}
}
}