Exemplo n.º 1
0
 public static bool IsFailedStatus(Job.StatusEnum status)
 {
     return status == StatusEnum.FailedExecution ||
            status == StatusEnum.FailedAbortOnServer ||
            status == StatusEnum.FailedToUploadServer ||
            status == StatusEnum.FailedToDownload ||
            status == StatusEnum.Failed;
 }
        public override void AddJob(Job job)
        {
            if (Jobs == null)
            {
                Jobs = new List<Job>();
            }
            Jobs.Add(job);

            handlersAdded.WaitOne();

            if (JobAdded != null)
            {
                JobAdded.Invoke((JobImpl)job, job.Status);
            }
        }
Exemplo n.º 3
0
        public bool EnqueueJob(
            string runCommand,
            string title,
            string testbenchName,
            string workingDirectory,
            ComComponent interpreter,
            Job.TypeEnum type = Job.TypeEnum.Command)
        {
            // TODO: cut down the number of input variables. interpreter and title should be enough
            try
            {
                JobServer manager;
                Job j = CreateJob(out manager);

                j.RunCommand = runCommand;
                j.Title = title;
                j.TestBenchName = testbenchName;
                j.WorkingDirectory = workingDirectory;
                j.Type = type;

                // TODO: allow empty Labels
                j.Labels = String.IsNullOrWhiteSpace(interpreter.result.Labels) ?
                    Job.DefaultLabels :
                    interpreter.result.Labels;

                j.BuildQuery = String.IsNullOrWhiteSpace(interpreter.result.BuildQuery) ?
                    Job.DefaultBuildQuery :
                    interpreter.result.BuildQuery;

                if (String.IsNullOrWhiteSpace(interpreter.result.ZippyServerSideHook) == false)
                {
                    j.ResultsZip = interpreter.result.ZippyServerSideHook as string;
                }

                jobsToAdd.Enqueue(new KeyValuePair<JobServer, Job>(manager, j));
                return true;

            }
            catch (System.Net.Sockets.SocketException ex)
            {
                throw new Exception("JobManager is not running. Please start it.", ex);
            }
            catch (Exception e)
            {
                // TODO: just log
                throw new Exception(string.Format("=> Job cannot be posted: {0}", e.ToString()));
            }
        }
Exemplo n.º 4
0
 public void JobStatusChangedSTA(Job job)
 {
     if (new Job.StatusEnum[] { Job.StatusEnum.WaitingForStart, Job.StatusEnum.Ready, Job.StatusEnum.QueuedLocal}.Contains(job.Status) == false)
     {
         var kvp = this.TestBenchJobMap.FirstOrDefault(x => x.Value == job);
         {
             foreach (var upstream in kvp.Key.UpstreamTestBenches)
             {
                 this.TestBenchJobMap[upstream].RerunEnabled = false;
             }
         }
     }
     if (job.IsFailed())
     {
         // TODO: how to indicate that the jobs are failed?
     }
     else if (job.Status == Job.StatusEnum.Succeeded)
     {
         // FIXME: could this fail??? would be nice to put it into a try catch.
         try
         {
             MonitorAndPostDownstreamTestBenches(job);
         }
         catch (JobFailure ex)
         {
             job.Status = Job.StatusEnum.Failed; // FIXME would be better to mark only downstream jobs as failed?
             File.WriteAllText(Path.Combine(job.WorkingDirectory, "_FAILED.txt"), ex.Message);
         }
     }
 }
Exemplo n.º 5
0
 public void JobStatusChanged(Job job, Job.StatusEnum status)
 {
     SoTTodo.Add(delegate { JobStatusChangedSTA(job); });
 }
Exemplo n.º 6
0
        private void MonitorAndPostDownstreamTestBenches(Job job)
        {
            var kvp = this.TestBenchJobMap.FirstOrDefault(x => x.Value == job);
            {
                foreach (var tbdown in kvp.Key.DownstreamTestBenches)
                {
                    if (tbdown.UpstreamTestBenches.All(x => this.TestBenchJobMap[x].Status == Job.StatusEnum.Succeeded))
                    {
                        var downJob = this.TestBenchJobMap[tbdown];
                        var jobInServer = this.Server.Jobs.FirstOrDefault(x => x.Id == downJob.Id);
                        if (jobInServer != null)
                        {
                            // already posted.
                            continue;
                        }
                        if (downJob.Status != Job.StatusEnum.Succeeded ||
                            downJob.IsFailed())
                        {
                            this.OpenProject();

                            try
                            {
                                // get all upstream test bench results back to the model.
                                GetResultsBackToModel(tbdown.UpstreamTestBenches);
                                // propagate the values between test benches
                                PropagateValueFlow();
                                // call interpreters for this test bench and post it to the job manager
                                RunTestBenchAndStart(tbdown);
                            }
                            finally
                            {
                                this.CloseProject();
                            }
                        }
                    }
                }
            }
        }
Exemplo n.º 7
0
        public override void Update()
        {
            base.Update();

            if (tradingPosts != null && tradingPosts.Count > 0)
            {
                ResourceManager.TradingPost tradingPost = tradingPosts[0];
                if (path.Count <= 0)
                {
                    MoveToTile(tradingPost.zeroPointTile, false);
                }
                if (overTile == tradingPost.zeroPointTile)
                {
                    foreach (ResourceManager.ReservedResources rr in tradingPost.GetInventory().TakeReservedResources(this))
                    {
                        foreach (ResourceManager.ResourceAmount ra in rr.resources)
                        {
                            caravan.GetInventory().ChangeResourceAmount(ra.resource, ra.amount, false);

                            ResourceManager.ConfirmedTradeResourceAmount confirmedTradeResourceAmount = caravan.confirmedResourcesToTrade.Find(crtt => crtt.resource == ra.resource);
                            if (confirmedTradeResourceAmount != null)
                            {
                                confirmedTradeResourceAmount.amountRemaining += ra.amount;
                            }
                        }
                    }
                    tradingPosts.RemoveAt(0);
                }
                if (tradingPosts.Count <= 0)
                {
                    JobManager.Job job = new JobManager.Job(
                        tradingPost.tile,
                        GameManager.resourceM.GetObjectPrefabByEnum(ResourceManager.ObjectEnum.CollectResources),
                        null,
                        0
                        )
                    {
                        transferResources = new List <ResourceManager.ResourceAmount>()
                    };
                    foreach (ResourceManager.ConfirmedTradeResourceAmount confirmedTradeResourceAmount in caravan.confirmedResourcesToTrade)
                    {
                        if (confirmedTradeResourceAmount.tradeAmount > 0)
                        {
                            tradingPost.GetInventory().ChangeResourceAmount(confirmedTradeResourceAmount.resource, confirmedTradeResourceAmount.tradeAmount, false);
                            confirmedTradeResourceAmount.amountRemaining = 0;
                            job.transferResources.Add(new ResourceManager.ResourceAmount(confirmedTradeResourceAmount.resource, confirmedTradeResourceAmount.tradeAmount));
                        }
                    }
                    caravan.confirmedResourcesToTrade.Clear();
                    GameManager.jobM.CreateJob(job);
                }
            }
            else
            {
                if (path.Count <= 0)
                {
                    Wander(caravan.targetTile, 4);
                }
                else
                {
                    wanderTimer = UnityEngine.Random.Range(10f, 20f);
                }
            }
        }
        private Job RunJob(Job job)
        {
            try
            {
                job.Status = Job.StatusEnum.QueuedLocal;
                SemJob[job.Type].WaitOne();
                SemAll.WaitOne();
                // n.b. this could race, but don't even start the job if shutdown is requested
                bool shutdownRequested = this.ShutdownPool.WaitOne(1);
                if (shutdownRequested)
                {
                    job.Status = Job.StatusEnum.FailedExecution;
                    return job;
                }

                System.Diagnostics.Process proc0 = new System.Diagnostics.Process();
                lock (processes)
                {
                    if (processes.ContainsKey(job))
                    {
                        processes.Remove(job);
                    }

                    processes.Add(job, proc0);
                }
                proc0.Exited += new EventHandler((o, args) =>
                {
                    lock (processes)
                    {
                        processes.Remove(job);
                    }
                });

                System.Diagnostics.ProcessStartInfo psi =
                    new System.Diagnostics.ProcessStartInfo();

                if (File.Exists(Path.Combine(job.WorkingDirectory, "testbench_manifest.json")))
                {
                    psi.FileName = META.VersionInfo.PythonVEnvExe;
                    psi.Arguments = "-m TestBenchExecutor testbench_manifest.json";
                }
                else
                {
                    psi.Arguments = "/S /C \"" + job.RunCommand + "\"";
                    psi.FileName = "cmd";
                }
                psi.UseShellExecute = false; // true
                psi.CreateNoWindow = true;
                psi.WindowStyle = System.Diagnostics.ProcessWindowStyle.Minimized;
                psi.WorkingDirectory = Path.Combine(job.WorkingDirectory);
                //psi.EnvironmentVariables["PATH"] = META.VersionInfo.PythonVEnvPath + "\\Scripts;" + System.Environment.GetEnvironmentVariable("PATH");

                psi.RedirectStandardOutput = true;
                psi.RedirectStandardError = true;
                proc0.StartInfo = psi;

                proc0.EnableRaisingEvents = true;
                ManualResetEvent processExited = new ManualResetEvent(false);
                proc0.Exited += (o, e) =>
                {
                    try
                    {
                        processExited.Set();
                    }
                    catch (System.ObjectDisposedException)
                    {
                    }
                };

                using (processExited)
                using (StreamWriter stderr = new StreamWriter(Path.Combine(job.WorkingDirectory, "stderr.txt")))
                using (StreamWriter stdout = new StreamWriter(Path.Combine(job.WorkingDirectory, "stdout.txt")))
                using (proc0)
                    try
                    {
                        proc0.OutputDataReceived += ((sender, e) =>
                        {
                            if (e.Data != null)
                            {
                                try
                                {
                                    stdout.WriteLine(e.Data);
                                }
                                catch (System.ObjectDisposedException)
                                {
                                }
                            }
                        });
                        proc0.ErrorDataReceived += ((sender, e) =>
                        {
                            if (e.Data != null)
                            {
                                try
                                {
                                    stderr.WriteLine(e.Data);
                                }
                                catch (System.ObjectDisposedException)
                                {
                                }
                            }
                        });
                        string failedLog = Path.Combine(job.WorkingDirectory, Failed);
                        File.Delete(failedLog);
                        proc0.Start();
                        job.Status = Job.StatusEnum.RunningLocal;
                        if (psi.RedirectStandardOutput)
                        {
                            // begin read only if it was redirected
                            proc0.BeginOutputReadLine();
                        }
                        if (psi.RedirectStandardError)
                        {
                            // begin read only if it was redirected
                            proc0.BeginErrorReadLine();
                        }
                        //JobStatusChanged.Invoke(job);

                        String commandToShowToUser;
                        if (psi.FileName == META.VersionInfo.PythonVEnvExe)
                        {
                            commandToShowToUser = META.VersionInfo.PythonVEnvExe;
                        }
                        else
                        {
                            commandToShowToUser = Path.Combine(job.WorkingDirectory, job.RunCommand);
                        }

                        int iWaitHandle = WaitHandle.WaitAny(new WaitHandle[] { processExited, this.ShutdownPool });
                        if (iWaitHandle == 1)
                        {
                            // JobManager is closing
                            try
                            {
                                proc0.Kill();
                            }
                            catch (System.InvalidOperationException)
                            {
                                // possible race between WaitAny and process exit
                            }
                            using (StreamWriter writer = new StreamWriter(failedLog))
                            {
                                writer.WriteLine("Execution was cancelled due to JobManager exit");
                            }
                            job.Status = Job.StatusEnum.FailedExecution;
                        }
                        else if (File.Exists(failedLog))
                        {
                            job.Status = Job.StatusEnum.Failed;
                            //JobStatusChanged.Invoke(job);
                        }
                        else if (proc0.ExitCode != 0)
                        {
                            using (StreamWriter writer = new StreamWriter(failedLog))
                            {
                                writer.WriteLine(String.Format("\"{0}\" exited with non-zero code {1}", commandToShowToUser, proc0.ExitCode));
                            }
                            job.Status = Job.StatusEnum.FailedExecution;
                        }
                    }
                    catch (Exception ex)
                    {
                        job.Status = Job.StatusEnum.Failed;
                        //JobStatusChanged.Invoke(job);
                        System.Diagnostics.Debug.WriteLine(ex);
                        Trace.TraceError(ex.ToString());
                    }
            }
            catch (Exception ex)
            {
                Trace.TraceError(ex.ToString());
            }
            finally
            {
                SemAll.Release();
                SemJob[job.Type].Release();
            }
            return job;
        }
        public void EnqueueJob(Job j)
        {
            Trace.TraceInformation(string.Format("JobEnqueued in local pool: {0} {1}", j.Id, j.Title));

            Task t = tf.StartNew(() => RunJob(j), ct).
                ContinueWith(job => Done(job));
            

            tasks.Add(t, j);
            
            //Task t = new Task<Job>(() => RunJob(j)).ContinueWith(job => Done(job));

        }
 public abstract void AddJob(Job job);
Exemplo n.º 11
0
        private void RunJob(Job job)
        {
            try
            {
                // n.b. this could race, but don't even start the job if shutdown is requested
                bool shutdownRequested = this.ShutdownPool.WaitOne(0);
                if (shutdownRequested)
                {
                    job.Status = Job.StatusEnum.FailedExecution;
                    return;
                }

                System.Diagnostics.Process proc0 = new System.Diagnostics.Process();

                System.Diagnostics.ProcessStartInfo psi =
                    new System.Diagnostics.ProcessStartInfo();

                psi.Arguments = "/S /C \"" + job.RunCommand + "\"";
                psi.CreateNoWindow = true;
                psi.FileName = "cmd";
                psi.UseShellExecute = false; // true
                psi.WindowStyle = System.Diagnostics.ProcessWindowStyle.Minimized;
                psi.WorkingDirectory = Path.Combine(job.WorkingDirectory);

                psi.RedirectStandardOutput = true;
                psi.RedirectStandardError = true;
                proc0.StartInfo = psi;

                proc0.EnableRaisingEvents = true;
                ManualResetEvent processExited = new ManualResetEvent(false);
                proc0.Exited += (o, e) =>
                {
                    try
                    {
                        processExited.Set();
                    }
                    catch (System.ObjectDisposedException)
                    {
                    }
                };

                using (processExited)
                using (StreamWriter stderr = new StreamWriter(Path.Combine(job.WorkingDirectory, "stderr.txt")))
                using (StreamWriter stdout = new StreamWriter(Path.Combine(job.WorkingDirectory, "stdout.txt")))
                using (proc0)
                    try
                    {
                        proc0.OutputDataReceived += ((sender, e) =>
                        {
                            if (e.Data != null)
                            {
                                try
                                {
                                    stdout.WriteLine(e.Data);
                                }
                                catch (System.ObjectDisposedException)
                                {
                                }
                            }
                        });
                        proc0.ErrorDataReceived += ((sender, e) =>
                        {
                            if (e.Data != null)
                            {
                                try
                                {
                                    stderr.WriteLine(e.Data);
                                }
                                catch (System.ObjectDisposedException)
                                {
                                }
                            }
                        });
                        string failedLog = Path.Combine(job.WorkingDirectory, Failed);
                        File.Delete(failedLog);
                        proc0.Start();
                        job.Status = Job.StatusEnum.RunningLocal;
                        if (psi.RedirectStandardOutput)
                        {
                            // begin read only if it was redirected
                            proc0.BeginOutputReadLine();
                        }
                        if (psi.RedirectStandardError)
                        {
                            // begin read only if it was redirected
                            proc0.BeginErrorReadLine();
                        }
                        //JobStatusChanged.Invoke(job);

                        int iWaitHandle = WaitHandle.WaitAny(new WaitHandle[] { processExited, this.ShutdownPool });
                        if (iWaitHandle == 1)
                        {
                            // JobManager is closing
                            try
                            {
                                proc0.Kill();
                            }
                            catch (System.InvalidOperationException)
                            {
                                // possible race between WaitAny and process exit
                            }
                            using (StreamWriter writer = new StreamWriter(failedLog))
                            {
                                writer.WriteLine("Execution was cancelled due to JobManager exit");
                            }
                            job.Status = Job.StatusEnum.FailedExecution;
                        }
                        else if (File.Exists(failedLog))
                        {
                            job.Status = Job.StatusEnum.Failed;
                            //JobStatusChanged.Invoke(job);
                        }
                        else if (proc0.ExitCode == 1)
                        {
                            // command not found
                            using (StreamWriter writer = new StreamWriter(failedLog))
                            {
                                writer.WriteLine("Specified command was not found: {0}",
                                    Path.Combine(job.WorkingDirectory, job.RunCommand));
                            }
                            job.Status = Job.StatusEnum.FailedExecution;
                            //JobStatusChanged.Invoke(job);
                        }
                        else if (proc0.ExitCode != 0)
                        {
                            using (StreamWriter writer = new StreamWriter(failedLog))
                            {
                                writer.WriteLine(String.Format("\"{0}\" exited with non-zero code {1}", job.RunCommand, proc0.ExitCode));
                            }
                            job.Status = Job.StatusEnum.FailedExecution;
                        }
                        else
                        {
                            job.Status = Job.StatusEnum.Succeeded;
                        }

                    }
                    catch (Exception ex)
                    {
                        job.Status = Job.StatusEnum.Failed;
                        //JobStatusChanged.Invoke(job);
                        System.Diagnostics.Debug.WriteLine(ex);
                        Trace.TraceError(ex.ToString());
                    }
            }
            catch (Exception ex)
            {
                Trace.TraceError(ex.ToString());
            }
            finally
            {
                lock (QueuedJobs)
                {
                    JobCapacities[job.Type]++;
                }
            }
        }
Exemplo n.º 12
0
        public void EnqueueJob(Job j)
        {
            Trace.TraceInformation(string.Format("JobEnqueued in local pool: {0} {1}", j.Id, j.Title));

            QueuedJobs.AddLast(j);
            JobAdded.Set();
        }