public void Clear()
{
lock (sync)
{
PendingJobs.Clear();
}
}
public async Task StartJob()
{
Job currentJob;
if (PendingJobs.NotEmpty() && !Cancelled)
{
lock (PendingJobs)
currentJob = PendingJobs.Dequeue();
lock (ScheduledJobs)
ScheduledJobs.Add(currentJob);
await jobSemaphore.WaitAsync();
if (ScheduledJobs.NotEmpty() && !Cancelled)
{
lock (ScheduledJobs)
ScheduledJobs.Remove(currentJob);
lock (ExecutingJobs)
ExecutingJobs.Add(currentJob);
await currentJob.Execute(ImageMaker);
if (!Cancelled)
{
lock (ExecutingJobs)
ExecutingJobs.Remove(currentJob);
}
jobSemaphore.Release();
}
}
}
/// <summary>
/// Searches for a <see cref="GeolocationBatchUpdateJob"/> by its id in every collection.
/// Returns null when not found.
/// </summary>
/// <param name="id">The <see cref="GeolocationBatchUpdateJob"/> identifier</param>
/// <returns>The <see cref="GeolocationBatchUpdateJob"/> or null if not found</returns>
private GeolocationBatchUpdateJob SearchJob(Guid id)
{
var registeredJobs = PendingJobs.Union(ExecutingJobs).Union(CompletedJobsReadOnly);
var job = registeredJobs.FirstOrDefault(x => x.Id == id);
return(job);
}
public void AddJob(Job job)
{
if (!PendingJobs.Contains(job))
{
lock (PendingJobs)
PendingJobs.Enqueue(job);
}
}
/// <summary>
/// Encapsulates the given <paramref name="request"/> within a <see cref="GeolocationBatchUpdateJob"/>
/// along with the <see cref="IPGeolocationProcessor"/> delegate to eventually invoke when
/// processing begins
/// </summary>
/// <param name="request">The <see cref="List{T}"/> of <see cref="IPGeolocationUpdateRequest"/></param>
/// <param name="processor">The <see cref="IPGeolocationProcessor"/> delegate to invoke at processing time</param>
/// <returns>
/// The assigned <see cref="Guid"/> associated with the newly created
/// <see cref="GeolocationBatchUpdateJob"/>
/// </returns>
public Guid AddJob(List <IPGeolocationUpdateRequest> request, IPGeolocationProcessor processor)
{
var backgroundJob = new GeolocationBatchUpdateJob(request, processor);
PendingJobs.Add(backgroundJob);
return(backgroundJob.Id);
}
private void ReceiveJobResult(PendingJob job, int[] resultData)
{
if (PendingJobs.Contains(job))
{
PendingJobs.Remove(job);
Tasker.CompletedJobs.Add(new Job(job.JobID, resultData));
}
}
/// <summary>
/// Attempts to queue pending <see cref="GeolocationBatchUpdateJob"/>s to executing based on
/// the available <see cref="JobBatchSize"/>
/// </summary>
private void TryQueuePendingJobsToExecuting()
{
if (ExecutingJobs.Count >= JobBatchSize)
{
return; // Skip queueing when full
}
var jobs = PendingJobs.Take(OpenJobBatchSlots);
MovePendingToExecuting(jobs);
}
public Job Dequeue()
{
lock (sync)
{
if (PendingJobs.Count == 0)
{
return(null);
}
CurrentJob = PendingJobs[0];
PendingJobs.Remove(CurrentJob);
return(CurrentJob);
}
}
public void EnqueueJob(Job j)
{
// TODO: ZIP the job instead of using placeholder
try
{
j.Status = Job.StatusEnum.ZippingPackage;
string zipFilename = ZipWorkingDirectory(j.WorkingDirectory);
j.Status = Job.StatusEnum.UploadPackage;
string zipArtifactId = "";
using (var fileStream = File.OpenRead(zipFilename))
{
zipArtifactId = Service.UploadArtifact(fileStream);
}
var runCommand = "";
if (File.Exists(Path.Combine(j.WorkingDirectory, "testbench_manifest.json")))
{
// TODO: How do we want to specify which Python to use?
//runCommand = "\"" + META.VersionInfo.PythonVEnvExe + "\"" +
runCommand = "python" +
" -m testbenchexecutor --detailed-errors testbench_manifest.json";
}
else
{
runCommand = j.RunCommand;
}
var relativeWorkingDirectory = new DirectoryInfo(j.WorkingDirectory).Name;
var labels = j.Labels != null ? j.Labels : "";
var newJobId = Service.CreateJob(runCommand, relativeWorkingDirectory, zipArtifactId, labels);
PendingJobs.Add(newJobId, j);
j.Status = Job.StatusEnum.PostedToServer;
}
catch (ZipFailedException e)
{
j.Status = Job.StatusEnum.Failed;
}
catch (RemoteExecutionService.RequestFailedException e)
{
j.Status = Job.StatusEnum.FailedToUploadServer;
}
catch (Exception)
{
j.Status = Job.StatusEnum.FailedToUploadServer;
}
}
public void Enqueue(Job job)
{
if (job.Priority < 6)
{
return;
}
lock (sync)
{
int insertPos = FindInsertPointForJob(job);
PendingJobs.Insert(insertPos, job);
if (PendingJobs.Count > QueueLimit)
{
PendingJobs.Remove(PendingJobs.Last());
}
}
}
public void GiveJob(int jobID, int[] data)
{
string dataString = "";
int workerID = ChooseWorker();
for (int i = 0; i < data.Length; i++)
{
dataString += data[i];
if (i != data.Length - 1)
{
dataString += '|';
}
}
string command = "job " + workerID + " " + jobID + " " + dataString + " \r\n";
PendingJobs.Add(new PendingJob(workerID, jobID));
ServerSocket.Send(Encoding.ASCII.GetBytes(command));
}
private void MonitorJobs()
{
while (ShutdownPool.WaitOne(0) == false)
{
foreach (var jobPair in PendingJobs)
{
try
{
var jobStatus = Service.GetJobInfo(jobPair.Key);
var job = jobPair.Value;
//TODO: on completion, fetch and extract the completed ZIP
if (IsJobStateCompleted(jobStatus.Status))
{
PendingJobs.Remove(jobPair.Key);
if (!string.IsNullOrEmpty(jobStatus.ResultZipId))
{
try
{
DownloadWorkspaceZip(job.WorkingDirectory, jobStatus.ResultZipId);
UnzipWorkingDirectory(job.WorkingDirectory);
}
catch (RemoteExecutionService.RequestFailedException)
{
job.Status = Job.StatusEnum.FailedToDownload;
}
catch (ZipFailedException)
{
job.Status = Job.StatusEnum.FailedToDownload;
}
catch (Exception)
{
job.Status = Job.StatusEnum.FailedToDownload;
}
}
}
string failedLog = Path.Combine(job.WorkingDirectory, LocalPool.Failed);
if (IsJobStateCompleted(jobStatus.Status) && File.Exists(failedLog))
{
job.Status = Job.StatusEnum.Failed;
}
else
{
switch (jobStatus.Status)
{
case RemoteExecutionService.RemoteJobState.Created:
job.Status = Job.StatusEnum.QueuedOnServer;
break;
case RemoteExecutionService.RemoteJobState.Running:
job.Status = Job.StatusEnum.RunningOnServer;
break;
case RemoteExecutionService.RemoteJobState.RequestingCancellation:
job.Status = Job.StatusEnum.AbortOnServerRequested;
break;
case RemoteExecutionService.RemoteJobState.Succeeded:
job.Status = Job.StatusEnum.Succeeded;
break;
case RemoteExecutionService.RemoteJobState.Failed:
job.Status = Job.StatusEnum.Failed;
break;
case RemoteExecutionService.RemoteJobState.Cancelled:
job.Status = Job.StatusEnum.FailedAbortOnServer;
break;
}
}
}
catch (RemoteExecutionService.RequestFailedException)
{
jobPair.Value.Status = Job.StatusEnum.Failed;
var job = jobPair.Key;
PendingJobs.Remove(jobPair.Key);
}
}
Thread.Sleep(1000);
}
}
/// <summary>
/// Moves the provided <see cref="GeolocationBatchUpdateJob"/> from pending to executing
/// </summary>
/// <param name="job">The <see cref="GeolocationBatchUpdateJob"/> to move</param>
private void MovePendingToExecuting(GeolocationBatchUpdateJob job)
{
PendingJobs.Remove(job);
ExecutingJobs.Add(job);
Buffer.Add(job.Id, job.Request as List <IPGeolocationUpdateRequest>);
}
