public static bool IsFailedStatus(Job.StatusEnum status)
{
return status == StatusEnum.FailedExecution ||
status == StatusEnum.FailedAbortOnServer ||
status == StatusEnum.FailedToUploadServer ||
status == StatusEnum.FailedToDownload ||
status == StatusEnum.Failed;
}
public override void AddJob(Job job)
{
if (Jobs == null)
{
Jobs = new List<Job>();
}
Jobs.Add(job);
handlersAdded.WaitOne();
if (JobAdded != null)
{
JobAdded.Invoke((JobImpl)job, job.Status);
}
}
public bool EnqueueJob(
string runCommand,
string title,
string testbenchName,
string workingDirectory,
ComComponent interpreter,
Job.TypeEnum type = Job.TypeEnum.Command)
{
// TODO: cut down the number of input variables. interpreter and title should be enough
try
{
JobServer manager;
Job j = CreateJob(out manager);
j.RunCommand = runCommand;
j.Title = title;
j.TestBenchName = testbenchName;
j.WorkingDirectory = workingDirectory;
j.Type = type;
// TODO: allow empty Labels
j.Labels = String.IsNullOrWhiteSpace(interpreter.result.Labels) ?
Job.DefaultLabels :
interpreter.result.Labels;
j.BuildQuery = String.IsNullOrWhiteSpace(interpreter.result.BuildQuery) ?
Job.DefaultBuildQuery :
interpreter.result.BuildQuery;
if (String.IsNullOrWhiteSpace(interpreter.result.ZippyServerSideHook) == false)
{
j.ResultsZip = interpreter.result.ZippyServerSideHook as string;
}
jobsToAdd.Enqueue(new KeyValuePair<JobServer, Job>(manager, j));
return true;
}
catch (System.Net.Sockets.SocketException ex)
{
throw new Exception("JobManager is not running. Please start it.", ex);
}
catch (Exception e)
{
// TODO: just log
throw new Exception(string.Format("=> Job cannot be posted: {0}", e.ToString()));
}
}
public void JobStatusChangedSTA(Job job)
{
if (new Job.StatusEnum[] { Job.StatusEnum.WaitingForStart, Job.StatusEnum.Ready, Job.StatusEnum.QueuedLocal}.Contains(job.Status) == false)
{
var kvp = this.TestBenchJobMap.FirstOrDefault(x => x.Value == job);
{
foreach (var upstream in kvp.Key.UpstreamTestBenches)
{
this.TestBenchJobMap[upstream].RerunEnabled = false;
}
}
}
if (job.IsFailed())
{
// TODO: how to indicate that the jobs are failed?
}
else if (job.Status == Job.StatusEnum.Succeeded)
{
// FIXME: could this fail??? would be nice to put it into a try catch.
try
{
MonitorAndPostDownstreamTestBenches(job);
}
catch (JobFailure ex)
{
job.Status = Job.StatusEnum.Failed; // FIXME would be better to mark only downstream jobs as failed?
File.WriteAllText(Path.Combine(job.WorkingDirectory, "_FAILED.txt"), ex.Message);
}
}
}
public void JobStatusChanged(Job job, Job.StatusEnum status)
{
SoTTodo.Add(delegate { JobStatusChangedSTA(job); });
}
private void MonitorAndPostDownstreamTestBenches(Job job)
{
var kvp = this.TestBenchJobMap.FirstOrDefault(x => x.Value == job);
{
foreach (var tbdown in kvp.Key.DownstreamTestBenches)
{
if (tbdown.UpstreamTestBenches.All(x => this.TestBenchJobMap[x].Status == Job.StatusEnum.Succeeded))
{
var downJob = this.TestBenchJobMap[tbdown];
var jobInServer = this.Server.Jobs.FirstOrDefault(x => x.Id == downJob.Id);
if (jobInServer != null)
{
// already posted.
continue;
}
if (downJob.Status != Job.StatusEnum.Succeeded ||
downJob.IsFailed())
{
this.OpenProject();
try
{
// get all upstream test bench results back to the model.
GetResultsBackToModel(tbdown.UpstreamTestBenches);
// propagate the values between test benches
PropagateValueFlow();
// call interpreters for this test bench and post it to the job manager
RunTestBenchAndStart(tbdown);
}
finally
{
this.CloseProject();
}
}
}
}
}
}
public override void Update()
{
base.Update();
if (tradingPosts != null && tradingPosts.Count > 0)
{
ResourceManager.TradingPost tradingPost = tradingPosts[0];
if (path.Count <= 0)
{
MoveToTile(tradingPost.zeroPointTile, false);
}
if (overTile == tradingPost.zeroPointTile)
{
foreach (ResourceManager.ReservedResources rr in tradingPost.GetInventory().TakeReservedResources(this))
{
foreach (ResourceManager.ResourceAmount ra in rr.resources)
{
caravan.GetInventory().ChangeResourceAmount(ra.resource, ra.amount, false);
ResourceManager.ConfirmedTradeResourceAmount confirmedTradeResourceAmount = caravan.confirmedResourcesToTrade.Find(crtt => crtt.resource == ra.resource);
if (confirmedTradeResourceAmount != null)
{
confirmedTradeResourceAmount.amountRemaining += ra.amount;
}
}
}
tradingPosts.RemoveAt(0);
}
if (tradingPosts.Count <= 0)
{
JobManager.Job job = new JobManager.Job(
tradingPost.tile,
GameManager.resourceM.GetObjectPrefabByEnum(ResourceManager.ObjectEnum.CollectResources),
null,
0
)
{
transferResources = new List <ResourceManager.ResourceAmount>()
};
foreach (ResourceManager.ConfirmedTradeResourceAmount confirmedTradeResourceAmount in caravan.confirmedResourcesToTrade)
{
if (confirmedTradeResourceAmount.tradeAmount > 0)
{
tradingPost.GetInventory().ChangeResourceAmount(confirmedTradeResourceAmount.resource, confirmedTradeResourceAmount.tradeAmount, false);
confirmedTradeResourceAmount.amountRemaining = 0;
job.transferResources.Add(new ResourceManager.ResourceAmount(confirmedTradeResourceAmount.resource, confirmedTradeResourceAmount.tradeAmount));
}
}
caravan.confirmedResourcesToTrade.Clear();
GameManager.jobM.CreateJob(job);
}
}
else
{
if (path.Count <= 0)
{
Wander(caravan.targetTile, 4);
}
else
{
wanderTimer = UnityEngine.Random.Range(10f, 20f);
}
}
}
private Job RunJob(Job job)
{
try
{
job.Status = Job.StatusEnum.QueuedLocal;
SemJob[job.Type].WaitOne();
SemAll.WaitOne();
// n.b. this could race, but don't even start the job if shutdown is requested
bool shutdownRequested = this.ShutdownPool.WaitOne(1);
if (shutdownRequested)
{
job.Status = Job.StatusEnum.FailedExecution;
return job;
}
System.Diagnostics.Process proc0 = new System.Diagnostics.Process();
lock (processes)
{
if (processes.ContainsKey(job))
{
processes.Remove(job);
}
processes.Add(job, proc0);
}
proc0.Exited += new EventHandler((o, args) =>
{
lock (processes)
{
processes.Remove(job);
}
});
System.Diagnostics.ProcessStartInfo psi =
new System.Diagnostics.ProcessStartInfo();
if (File.Exists(Path.Combine(job.WorkingDirectory, "testbench_manifest.json")))
{
psi.FileName = META.VersionInfo.PythonVEnvExe;
psi.Arguments = "-m TestBenchExecutor testbench_manifest.json";
}
else
{
psi.Arguments = "/S /C \"" + job.RunCommand + "\"";
psi.FileName = "cmd";
}
psi.UseShellExecute = false; // true
psi.CreateNoWindow = true;
psi.WindowStyle = System.Diagnostics.ProcessWindowStyle.Minimized;
psi.WorkingDirectory = Path.Combine(job.WorkingDirectory);
//psi.EnvironmentVariables["PATH"] = META.VersionInfo.PythonVEnvPath + "\\Scripts;" + System.Environment.GetEnvironmentVariable("PATH");
psi.RedirectStandardOutput = true;
psi.RedirectStandardError = true;
proc0.StartInfo = psi;
proc0.EnableRaisingEvents = true;
ManualResetEvent processExited = new ManualResetEvent(false);
proc0.Exited += (o, e) =>
{
try
{
processExited.Set();
}
catch (System.ObjectDisposedException)
{
}
};
using (processExited)
using (StreamWriter stderr = new StreamWriter(Path.Combine(job.WorkingDirectory, "stderr.txt")))
using (StreamWriter stdout = new StreamWriter(Path.Combine(job.WorkingDirectory, "stdout.txt")))
using (proc0)
try
{
proc0.OutputDataReceived += ((sender, e) =>
{
if (e.Data != null)
{
try
{
stdout.WriteLine(e.Data);
}
catch (System.ObjectDisposedException)
{
}
}
});
proc0.ErrorDataReceived += ((sender, e) =>
{
if (e.Data != null)
{
try
{
stderr.WriteLine(e.Data);
}
catch (System.ObjectDisposedException)
{
}
}
});
string failedLog = Path.Combine(job.WorkingDirectory, Failed);
File.Delete(failedLog);
proc0.Start();
job.Status = Job.StatusEnum.RunningLocal;
if (psi.RedirectStandardOutput)
{
// begin read only if it was redirected
proc0.BeginOutputReadLine();
}
if (psi.RedirectStandardError)
{
// begin read only if it was redirected
proc0.BeginErrorReadLine();
}
//JobStatusChanged.Invoke(job);
String commandToShowToUser;
if (psi.FileName == META.VersionInfo.PythonVEnvExe)
{
commandToShowToUser = META.VersionInfo.PythonVEnvExe;
}
else
{
commandToShowToUser = Path.Combine(job.WorkingDirectory, job.RunCommand);
}
int iWaitHandle = WaitHandle.WaitAny(new WaitHandle[] { processExited, this.ShutdownPool });
if (iWaitHandle == 1)
{
// JobManager is closing
try
{
proc0.Kill();
}
catch (System.InvalidOperationException)
{
// possible race between WaitAny and process exit
}
using (StreamWriter writer = new StreamWriter(failedLog))
{
writer.WriteLine("Execution was cancelled due to JobManager exit");
}
job.Status = Job.StatusEnum.FailedExecution;
}
else if (File.Exists(failedLog))
{
job.Status = Job.StatusEnum.Failed;
//JobStatusChanged.Invoke(job);
}
else if (proc0.ExitCode != 0)
{
using (StreamWriter writer = new StreamWriter(failedLog))
{
writer.WriteLine(String.Format("\"{0}\" exited with non-zero code {1}", commandToShowToUser, proc0.ExitCode));
}
job.Status = Job.StatusEnum.FailedExecution;
}
}
catch (Exception ex)
{
job.Status = Job.StatusEnum.Failed;
//JobStatusChanged.Invoke(job);
System.Diagnostics.Debug.WriteLine(ex);
Trace.TraceError(ex.ToString());
}
}
catch (Exception ex)
{
Trace.TraceError(ex.ToString());
}
finally
{
SemAll.Release();
SemJob[job.Type].Release();
}
return job;
}
public void EnqueueJob(Job j)
{
Trace.TraceInformation(string.Format("JobEnqueued in local pool: {0} {1}", j.Id, j.Title));
Task t = tf.StartNew(() => RunJob(j), ct).
ContinueWith(job => Done(job));
tasks.Add(t, j);
//Task t = new Task<Job>(() => RunJob(j)).ContinueWith(job => Done(job));
}
public abstract void AddJob(Job job);
private void RunJob(Job job)
{
try
{
// n.b. this could race, but don't even start the job if shutdown is requested
bool shutdownRequested = this.ShutdownPool.WaitOne(0);
if (shutdownRequested)
{
job.Status = Job.StatusEnum.FailedExecution;
return;
}
System.Diagnostics.Process proc0 = new System.Diagnostics.Process();
System.Diagnostics.ProcessStartInfo psi =
new System.Diagnostics.ProcessStartInfo();
psi.Arguments = "/S /C \"" + job.RunCommand + "\"";
psi.CreateNoWindow = true;
psi.FileName = "cmd";
psi.UseShellExecute = false; // true
psi.WindowStyle = System.Diagnostics.ProcessWindowStyle.Minimized;
psi.WorkingDirectory = Path.Combine(job.WorkingDirectory);
psi.RedirectStandardOutput = true;
psi.RedirectStandardError = true;
proc0.StartInfo = psi;
proc0.EnableRaisingEvents = true;
ManualResetEvent processExited = new ManualResetEvent(false);
proc0.Exited += (o, e) =>
{
try
{
processExited.Set();
}
catch (System.ObjectDisposedException)
{
}
};
using (processExited)
using (StreamWriter stderr = new StreamWriter(Path.Combine(job.WorkingDirectory, "stderr.txt")))
using (StreamWriter stdout = new StreamWriter(Path.Combine(job.WorkingDirectory, "stdout.txt")))
using (proc0)
try
{
proc0.OutputDataReceived += ((sender, e) =>
{
if (e.Data != null)
{
try
{
stdout.WriteLine(e.Data);
}
catch (System.ObjectDisposedException)
{
}
}
});
proc0.ErrorDataReceived += ((sender, e) =>
{
if (e.Data != null)
{
try
{
stderr.WriteLine(e.Data);
}
catch (System.ObjectDisposedException)
{
}
}
});
string failedLog = Path.Combine(job.WorkingDirectory, Failed);
File.Delete(failedLog);
proc0.Start();
job.Status = Job.StatusEnum.RunningLocal;
if (psi.RedirectStandardOutput)
{
// begin read only if it was redirected
proc0.BeginOutputReadLine();
}
if (psi.RedirectStandardError)
{
// begin read only if it was redirected
proc0.BeginErrorReadLine();
}
//JobStatusChanged.Invoke(job);
int iWaitHandle = WaitHandle.WaitAny(new WaitHandle[] { processExited, this.ShutdownPool });
if (iWaitHandle == 1)
{
// JobManager is closing
try
{
proc0.Kill();
}
catch (System.InvalidOperationException)
{
// possible race between WaitAny and process exit
}
using (StreamWriter writer = new StreamWriter(failedLog))
{
writer.WriteLine("Execution was cancelled due to JobManager exit");
}
job.Status = Job.StatusEnum.FailedExecution;
}
else if (File.Exists(failedLog))
{
job.Status = Job.StatusEnum.Failed;
//JobStatusChanged.Invoke(job);
}
else if (proc0.ExitCode == 1)
{
// command not found
using (StreamWriter writer = new StreamWriter(failedLog))
{
writer.WriteLine("Specified command was not found: {0}",
Path.Combine(job.WorkingDirectory, job.RunCommand));
}
job.Status = Job.StatusEnum.FailedExecution;
//JobStatusChanged.Invoke(job);
}
else if (proc0.ExitCode != 0)
{
using (StreamWriter writer = new StreamWriter(failedLog))
{
writer.WriteLine(String.Format("\"{0}\" exited with non-zero code {1}", job.RunCommand, proc0.ExitCode));
}
job.Status = Job.StatusEnum.FailedExecution;
}
else
{
job.Status = Job.StatusEnum.Succeeded;
}
}
catch (Exception ex)
{
job.Status = Job.StatusEnum.Failed;
//JobStatusChanged.Invoke(job);
System.Diagnostics.Debug.WriteLine(ex);
Trace.TraceError(ex.ToString());
}
}
catch (Exception ex)
{
Trace.TraceError(ex.ToString());
}
finally
{
lock (QueuedJobs)
{
JobCapacities[job.Type]++;
}
}
}
public void EnqueueJob(Job j)
{
Trace.TraceInformation(string.Format("JobEnqueued in local pool: {0} {1}", j.Id, j.Title));
QueuedJobs.AddLast(j);
JobAdded.Set();
}