Пример #1
0
        static void Main(string[] args)
        {
            Console.Write("hpc: ");
            string serverName = Console.ReadLine();

            Console.Write("user: "******"job template (<Enter> - 'default'): ");
            string jobTemplate = Console.ReadLine();

            if (string.IsNullOrWhiteSpace(jobTemplate))
            {
                jobTemplate = "Default";
            }

            IScheduler scheduler = new Scheduler();

            scheduler.Connect(serverName);
            ISchedulerJob job = scheduler.CreateJob();

            job.SetJobTemplate(jobTemplate);
            ISchedulerTask task = job.CreateTask();

            task.CommandLine = "dir";
            job.AddTask(task);

            // Specify the events that you want to receive.
            job.OnJobState  += JobStateCallback;
            job.OnTaskState += TaskStateCallback;

            // Start the job.
            scheduler.SubmitJob(job, user, null);
            manualEvent.WaitOne();
        }
Пример #2
0
        private static void SubmitViaAPI3(ClusterSubmitterArgs clusterArgs, IDistributable distributableObj)
        {
            Console.WriteLine(string.Format("Connecting to cluster {0} using API version 3 .", clusterArgs.Cluster));

            using (IScheduler scheduler = new Scheduler())
            {
                scheduler.Connect(clusterArgs.Cluster);
                ISchedulerJob job = scheduler.CreateJob();

                job.Name     = distributableObj.JobName;
                job.Priority = clusterArgs.Priority;

                if (clusterArgs.JobTemplate != null)
                {
                    Microsoft.Hpc.Scheduler.IStringCollection jobTemplates = scheduler.GetJobTemplateList();
                    string decodedJobTemplate = System.Web.HttpUtility.UrlDecode(clusterArgs.JobTemplate);
                    if (jobTemplates.Contains(decodedJobTemplate))
                    {
                        job.SetJobTemplate(decodedJobTemplate);
                    }
                    else
                    {
                        Console.WriteLine(string.Format(Resource.Job_template, decodedJobTemplate));
                        foreach (var template in jobTemplates)
                        {
                            Console.Write("'" + template + "' ");
                        }
                        Console.WriteLine(Resource.SubmitViaAPI3);
                    }
                }


                if (clusterArgs.NumCoresPerTask != null)
                {
                    clusterArgs.IsExclusive = false;
                }

                IStringCollection nodesToUse = null;

                if (clusterArgs.NodeExclusionList != null && clusterArgs.NodeExclusionList.Count > 0)
                {
                    nodesToUse = GetNodesToUse(clusterArgs, scheduler, job);
                }
                else if (clusterArgs.NodesToUseList != null && clusterArgs.NodesToUseList.Count > 0)
                {
                    nodesToUse = scheduler.CreateStringCollection();
                    foreach (string nodeName in clusterArgs.NodesToUseList)
                    {
                        nodesToUse.Add(nodeName);
                    }
                }
                else if (clusterArgs.NumCoresPerTask != null)
                {
                    job.AutoCalculateMax = true;
                    job.AutoCalculateMin = true;
                }
                else if (clusterArgs.IsExclusive)
                {
                    job.UnitType = Microsoft.Hpc.Scheduler.Properties.JobUnitType.Node;
                    if (clusterArgs.MinimumNumberOfNodes != null)
                    {
                        job.MaximumNumberOfNodes = clusterArgs.MaximumNumberOfNodes.Value;
                        job.MinimumNumberOfNodes = clusterArgs.MinimumNumberOfNodes.Value;
                    }
                }
                else if (clusterArgs.MinimumNumberOfCores != null)
                {
                    if (clusterArgs.MaximumNumberOfCores == null)
                    {
                        job.AutoCalculateMax = true;
                    }
                    else
                    {
                        job.AutoCalculateMax     = false;
                        job.MaximumNumberOfCores = clusterArgs.MaximumNumberOfCores.Value;
                    }
                    job.MaximumNumberOfCores = clusterArgs.MaximumNumberOfCores ?? Math.Max(clusterArgs.TaskCount, scheduler.GetCounters().TotalCores);
                    job.MinimumNumberOfCores = clusterArgs.MinimumNumberOfCores.Value;
                    job.AutoCalculateMin     = false;
                }
                else
                {
                    job.AutoCalculateMax = true;
                    job.AutoCalculateMin = true;
                }

                if (!clusterArgs.OnlyDoCleanup)
                {
                    if (clusterArgs.TaskRange.IsContiguous())
                    {
                        if (clusterArgs.TaskRange.LastElement > clusterArgs.TaskCount - 1)
                        {
                            clusterArgs.TaskRange = new RangeCollection(clusterArgs.TaskRange.FirstElement, clusterArgs.TaskCount - 1);
                        }
                        ISchedulerTask task = CreateTask(null, clusterArgs, job, distributableObj, nodesToUse);

                        task.Type = TaskType.ParametricSweep;

                        task.StartValue = 0;
                        task.EndValue   = clusterArgs.TaskCount - 1;

                        job.AddTask(task);
                    }
                    else
                    {
                        job.AddTasks(clusterArgs.TaskRange.Select(taskNum => CreateTask((int)taskNum, clusterArgs, job, distributableObj, nodesToUse)).ToArray());
                    }
                }
                else
                {
                    clusterArgs.Cleanup = true;
                }

                ISchedulerTask cleanupTask = null;
                if (clusterArgs.Cleanup)
                {
                    cleanupTask = AddCleanupTaskToJob(clusterArgs, scheduler, job, distributableObj);
                }

                Console.WriteLine(Resource.Submitting_job);
                scheduler.SubmitJob(job, null, null);
                clusterArgs.JobID = job.Id;
                Console.WriteLine(job.Name + Resource.submitted);
            }
        }
Пример #3
0
        /// <summary>
        /// This method will only be internally called by session launcher,
        /// before the job is submitted. It combines the user's job properties and the
        /// items set in the service start info. The job properties will override the start info items.
        /// </summary>
        /// <param name="startInfo">the session start info</param>
        /// <param name="schedulerJob">service job</param>
        /// <param name="traceLevel">diag trace level</param>
        internal static void MakeJobProperties(SessionStartInfoContract startInfo, ISchedulerJob schedulerJob, string traceLevel)
        {
            Debug.Assert(startInfo != null,
                         "The session startInfo cannot be null.");

            Debug.Assert(!string.IsNullOrEmpty(startInfo.ServiceName),
                         "The service name in the session start info cannnot be null or empty.");

            schedulerJob.ServiceName = startInfo.ServiceName;
            schedulerJob.SetEnvironmentVariable("CCP_SERVICENAME", startInfo.ServiceName);
            if (startInfo.CanPreempt.HasValue)
            {
                schedulerJob.CanPreempt = startInfo.CanPreempt.Value;
            }

            if (!string.IsNullOrEmpty(startInfo.JobTemplate))
            {
                schedulerJob.SetJobTemplate(startInfo.JobTemplate);
            }

            if (startInfo.Priority.HasValue)
            {
                schedulerJob.Priority = (JobPriority)startInfo.Priority.Value;
            }

            if (startInfo.ExtendedPriority.HasValue)
            {
                schedulerJob.ExpandedPriority = startInfo.ExtendedPriority.Value;
            }

            schedulerJob.Progress = 0;

            // For max units
            if (startInfo.MaxUnits != null)
            {
                schedulerJob.MaximumNumberOfCores         =
                    schedulerJob.MaximumNumberOfSockets   =
                        schedulerJob.MaximumNumberOfNodes = startInfo.MaxUnits.Value;

                schedulerJob.AutoCalculateMax = false;
            }

            // For min units
            if (startInfo.MinUnits != null)
            {
                schedulerJob.MinimumNumberOfCores         =
                    schedulerJob.MinimumNumberOfSockets   =
                        schedulerJob.MinimumNumberOfNodes = startInfo.MinUnits.Value;

                schedulerJob.AutoCalculateMin = false;
            }

            // Should set UnitType after above resource count update
            if (startInfo.ResourceUnitType != null)
            {
                schedulerJob.UnitType = (JobUnitType)startInfo.ResourceUnitType.Value;
            }

            schedulerJob.Name = string.IsNullOrEmpty(startInfo.ServiceJobName) ?
                                string.Format("{0} - WCF service", startInfo.ServiceName) :
                                startInfo.ServiceJobName;

            if (!string.IsNullOrEmpty(startInfo.ServiceJobProject))
            {
                schedulerJob.Project = startInfo.ServiceJobProject;
            }

            if (!string.IsNullOrEmpty(startInfo.NodeGroupsStr))
            {
                string[] nodes = startInfo.NodeGroupsStr.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
                foreach (string node in nodes)
                {
                    schedulerJob.NodeGroups.Add(node);
                }
            }

            if (!string.IsNullOrEmpty(startInfo.RequestedNodesStr))
            {
                schedulerJob.RequestedNodes = new StringCollection(startInfo.RequestedNodesStr.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries));
            }

            if (startInfo.Runtime >= 0)
            {
                schedulerJob.Runtime = startInfo.Runtime;
            }

            // start adding the broker settings.
            schedulerJob.SetCustomProperty(BrokerSettingsConstants.ShareSession, startInfo.ShareSession.ToString());
            schedulerJob.SetCustomProperty(BrokerSettingsConstants.UseAad, startInfo.UseAad.ToString());
            schedulerJob.SetCustomProperty(BrokerSettingsConstants.Secure, startInfo.Secure.ToString());
            schedulerJob.SetCustomProperty(BrokerSettingsConstants.TransportScheme, ((int)startInfo.TransportScheme).ToString());
            schedulerJob.SetCustomProperty(BrokerSettingsConstants.UseAzureQueue, (startInfo.UseAzureQueue == true).ToString());
            schedulerJob.SetCustomProperty(BrokerSettingsConstants.LocalUser, (startInfo.LocalUser == true).ToString());

            var context   = HpcContext.Get();
            var principal = Thread.CurrentPrincipal;

            if (principal.IsHpcAadPrincipal(context))
            {
                string identity = principal.GenerateSecurityIdentifierFromAadPrincipal(context).Value + ";" + principal.Identity.Name;
                schedulerJob.SetCustomProperty(BrokerSettingsConstants.AadUserIdentity, identity);
            }

            // Save ServiceVersion if set
            if (startInfo.ServiceVersion != null)
            {
                schedulerJob.SetCustomProperty(BrokerSettingsConstants.ServiceVersion, startInfo.ServiceVersion.ToString());
            }

            string[] customPropNames = new string[]
            {
                BrokerSettingsConstants.AllocationGrowLoadRatioThreshold,
                BrokerSettingsConstants.AllocationShrinkLoadRatioThreshold,
                BrokerSettingsConstants.ClientIdleTimeout,
                BrokerSettingsConstants.SessionIdleTimeout,
                BrokerSettingsConstants.MessagesThrottleStartThreshold,
                BrokerSettingsConstants.MessagesThrottleStopThreshold,
                BrokerSettingsConstants.ClientConnectionTimeout,
                BrokerSettingsConstants.ServiceConfigMaxMessageSize,
                BrokerSettingsConstants.ServiceConfigOperationTimeout,
                BrokerSettingsConstants.DispatcherCapacityInGrowShrink
            };

            int?[] intNullableValues = new int?[]
            {
                startInfo.AllocationGrowLoadRatioThreshold,
                startInfo.AllocationShrinkLoadRatioThreshold,
                startInfo.ClientIdleTimeout,
                startInfo.SessionIdleTimeout,
                startInfo.MessagesThrottleStartThreshold,
                startInfo.MessagesThrottleStopThreshold,
                startInfo.ClientConnectionTimeout,
                startInfo.MaxMessageSize,
                startInfo.ServiceOperationTimeout,
                startInfo.DispatcherCapacityInGrowShrink
            };

            Debug.Assert(intNullableValues.Length == customPropNames.Length);

            for (int i = 0; i < customPropNames.Length; i++)
            {
                if (intNullableValues[i].HasValue)
                {
                    schedulerJob.SetCustomProperty(customPropNames[i], intNullableValues[i].Value.ToString());
                }
            }

            // add soa diag settings
            schedulerJob.SetCustomProperty(BrokerSettingsConstants.SoaDiagTraceLevel, traceLevel);
            schedulerJob.SetCustomProperty(BrokerSettingsConstants.SoaDiagTraceCleanup, Boolean.FalseString);
        }
Пример #4
0
        public void SubmitHPCJob(string db, bool isNew, int newID, string cluster, string nodegroup, int priority,
                                 string locality, string limitsMin, string limitsMax, string sharedDir,
                                 string executor,
                                 string jobTemplate,
                                 int jobTimeout, int taskTimeout,
                                 int nworkers = 0)
        {
            string limitsMinTrimmed = limitsMin.Trim();
            string limitsMaxTrimmed = limitsMax.Trim();

            SqlConnection sql = Connect(db);
            SqlCommand    cmd = null;

            scheduler.Connect(cluster);
            ISchedulerJob hpcJob = scheduler.CreateJob();

            if (jobTemplate != null)
            {
                hpcJob.SetJobTemplate(jobTemplate);
            }
            if (jobTimeout != 0)
            {
                hpcJob.Runtime = jobTimeout;
            }
            hpcJob.FailOnTaskFailure = false;

            try
            {
                if (nodegroup != "<Any>")
                {
                    hpcJob.NodeGroups.Add(nodegroup);
                }
                hpcJob.Name        = "Z3 Performance Test (" + newID + ")";
                hpcJob.IsExclusive = true;
                hpcJob.CanPreempt  = true;
                SetPriority(hpcJob, priority);
                hpcJob.Project = "Z3";

                if (locality == "Socket")
                {
                    hpcJob.UnitType = JobUnitType.Socket;
                }
                else if (locality == "Core")
                {
                    hpcJob.UnitType = JobUnitType.Core;
                }
                else if (locality == "Node")
                {
                    hpcJob.UnitType = JobUnitType.Node;
                }
                else
                {
                    throw new Exception("Unknown locality.");
                }

                uint min = 0;
                uint max = 0;

                if (limitsMinTrimmed != "")
                {
                    try { min = Convert.ToUInt32(limitsMinTrimmed); }
                    catch { min = 0; }
                }
                if (limitsMax != "")
                {
                    try { max = Convert.ToUInt32(limitsMaxTrimmed); }
                    catch { max = 0; }
                }

                ISchedulerCounters ctrs = scheduler.GetCounters();
                if (locality == "Socket")
                {
                    if (min > 0)
                    {
                        hpcJob.MinimumNumberOfSockets = (int)min;
                    }
                    max = ((max == 0) ? (uint)ctrs.TotalSockets: max);
                    hpcJob.MaximumNumberOfSockets = (int)max;
                }
                else if (locality == "Core")
                {
                    if (min > 0)
                    {
                        hpcJob.MinimumNumberOfCores = (int)min;
                    }
                    max = ((max == 0) ? (uint)ctrs.TotalCores : max);
                    hpcJob.MaximumNumberOfCores = (int)max;
                }
                else if (locality == "Node")
                {
                    if (min > 0)
                    {
                        hpcJob.MinimumNumberOfNodes = (int)min;
                    }
                    max = ((max == 0) ? (uint)ctrs.TotalNodes: max);
                    hpcJob.MaximumNumberOfNodes = (int)max;
                }

                uint progressTotal = max + 3;

                // Add population task.
                if (WorkerReportsProgress)
                {
                    ReportProgress(Convert.ToInt32(100.0 * 1 / (double)max));
                }
                ISchedulerTask populateTask = hpcJob.CreateTask();
                SetResources(populateTask, locality);
                populateTask.IsRerunnable = false;
                populateTask.IsExclusive  = false;
                // populateTask.WorkDirectory = sharedDir;
                //populateTask.CommandLine = executor + " " + newID + " ? \"" + db + "\"";
                populateTask.CommandLine = "pushd " + sharedDir + " & " + Path.GetFileName(executor) + " " + newID + " ? \"" + db + "\"";
                populateTask.Name        = "Populate";
                if (taskTimeout != 0)
                {
                    populateTask.Runtime = taskTimeout;
                }
                populateTask.FailJobOnFailure = true;
                hpcJob.AddTask(populateTask);

                for (int i = 0; i < max; i++)
                {
                    // Add worker task.
                    if (WorkerReportsProgress)
                    {
                        ReportProgress(Convert.ToInt32(100.0 * (i + 1) / (double)max));
                    }
                    ISchedulerTask task = hpcJob.CreateTask();
                    SetResources(task, locality);
                    // task.WorkDirectory = sharedDir;
                    // task.CommandLine = executor + " " + newID + " \"" + db + "\"";
                    task.CommandLine  = "pushd " + sharedDir + " & " + Path.GetFileName(executor) + " " + newID + " \"" + db + "\"";
                    task.IsExclusive  = false;
                    task.IsRerunnable = true;
                    task.DependsOn.Add("Populate");
                    task.Name = "Worker";
                    if (taskTimeout != 0)
                    {
                        task.Runtime = taskTimeout;
                    }
                    populateTask.FailJobOnFailure = false;
                    hpcJob.AddTask(task);
                }

                // Add recovery task.
                if (WorkerReportsProgress)
                {
                    ReportProgress(Convert.ToInt32(100.0 * (progressTotal - 1) / (double)max));
                }
                ISchedulerTask rTask = hpcJob.CreateTask();
                SetResources(rTask, locality);
                rTask.IsRerunnable = true;
                rTask.IsExclusive  = false;
                // rTask.WorkDirectory = sharedDir;
                // rTask.CommandLine = executor + " " + newID + " ! \"" + db + "\"";
                rTask.CommandLine = "pushd " + sharedDir + " & " + Path.GetFileName(executor) + " " + newID + " ! \"" + db + "\"";
                rTask.DependsOn.Add("Worker");
                rTask.Name = "Recovery";
                if (taskTimeout != 0)
                {
                    rTask.Runtime = taskTimeout;
                }
                rTask.FailJobOnFailure = true;
                hpcJob.AddTask(rTask);

                // Add deletion task.
                if (WorkerReportsProgress)
                {
                    ReportProgress(Convert.ToInt32(100.0 * (progressTotal) / (double)max));
                }
                ISchedulerTask dTask = hpcJob.CreateTask();
                SetResources(dTask, locality);
                dTask.IsRerunnable = true;
                dTask.IsExclusive  = false;
                // dTask.WorkDirectory = sharedDir;
                // dTask.CommandLine = "del " + sharedDir + "\\" + executor;
                dTask.CommandLine = "pushd " + sharedDir + " & del " + Path.GetFileName(executor);
                dTask.Name        = "Delete worker";
                dTask.DependsOn.Add("Recovery");
                if (taskTimeout != 0)
                {
                    dTask.Runtime = taskTimeout;
                }
                dTask.FailJobOnFailure = false;
                hpcJob.AddTask(dTask);

                scheduler.AddJob(hpcJob);
                scheduler.SubmitJob(hpcJob, null, null);

                if (isNew)
                {
                    cmd = new SqlCommand("UPDATE Experiments SET ClusterJobID=" + hpcJob.Id.ToString() + " WHERE ID=" + newID.ToString() + "; ", sql);
                    cmd.CommandTimeout = 0;
                    cmd.ExecuteNonQuery();
                }
            }
            catch (Exception ex)
            {
                cmd = new SqlCommand("DELETE FROM JobQueue WHERE ExperimentID=" + newID + "; DELETE FROM Experiments WHERE ID=" + newID, sql);
                cmd.CommandTimeout = 0;
                cmd.ExecuteNonQuery();
                if (hpcJob.State == JobState.Configuring ||
                    hpcJob.State == JobState.ExternalValidation ||
                    hpcJob.State == JobState.Queued ||
                    hpcJob.State == JobState.Running ||
                    hpcJob.State == JobState.Submitted ||
                    hpcJob.State == JobState.Validating)
                {
                    try { scheduler.CancelJob(hpcJob.Id, "Aborted."); }
                    catch (Exception) { }
                }
                throw ex;
            }

            //return totalJobs;
        }
Пример #5
0
        public void SubmitCatchall(string db, string cluster, string locality, int priority, string nodegroup, string executor, string min, string max, string jobTemplate, int jobTimeout, int taskTimeout)
        {
            scheduler.Connect(cluster);
            ISchedulerJob hpcJob = scheduler.CreateJob();

            if (jobTemplate != null)
            {
                hpcJob.SetJobTemplate(jobTemplate);
            }
            if (jobTimeout != 0)
            {
                hpcJob.Runtime = jobTimeout;
            }
            try
            {
                if (nodegroup != "<Any>")
                {
                    hpcJob.NodeGroups.Add(nodegroup);
                }
                hpcJob.Name        = "Z3 Performance Test (catchall)";
                hpcJob.IsExclusive = true;
                hpcJob.CanPreempt  = true;
                SetPriority(hpcJob, priority);
                hpcJob.Project = "Z3";

                uint   fmin             = 0;
                uint   fmax             = 0;
                string limitsMinTrimmed = min.Trim();
                string limitsMaxTrimmed = max.Trim();

                if (limitsMinTrimmed != "")
                {
                    try { fmin = Convert.ToUInt32(limitsMinTrimmed); }
                    catch { fmin = 0; }
                }
                if (limitsMaxTrimmed != "")
                {
                    try { fmax = Convert.ToUInt32(limitsMaxTrimmed); }
                    catch { fmax = 0; }
                }

                ISchedulerCounters ctrs = scheduler.GetCounters();
                if (locality == "Socket")
                {
                    hpcJob.UnitType = JobUnitType.Socket;
                    if (fmin > 0)
                    {
                        hpcJob.MinimumNumberOfSockets = (int)fmin;
                    }
                    fmax = ((fmax == 0) ? (uint)ctrs.TotalSockets : fmax);
                    hpcJob.MaximumNumberOfSockets = (int)fmax;
                }
                else if (locality == "Core")
                {
                    hpcJob.UnitType = JobUnitType.Core;
                    if (fmin > 0)
                    {
                        hpcJob.MinimumNumberOfCores = (int)fmin;
                    }
                    fmax = ((fmax == 0) ? (uint)ctrs.TotalCores : fmax);
                    hpcJob.MaximumNumberOfCores = (int)fmax;
                }
                else if (locality == "Node")
                {
                    hpcJob.UnitType = JobUnitType.Node;
                    if (fmin > 0)
                    {
                        hpcJob.MinimumNumberOfNodes = (int)fmin;
                    }
                    fmax = ((fmax == 0) ? (uint)ctrs.TotalNodes : fmax);
                    hpcJob.MaximumNumberOfNodes = (int)fmax;
                }

                for (int i = 0; i < fmax; i++)
                {
                    // Add worker task.
                    if (WorkerReportsProgress)
                    {
                        ReportProgress(Convert.ToInt32(100.0 * (i + 1) / (double)fmax));
                    }
                    ISchedulerTask task = hpcJob.CreateTask();
                    SetResources(task, locality);
                    task.WorkDirectory = Path.GetDirectoryName(Path.GetFullPath(executor));
                    task.CommandLine   = "pushd " + Path.GetDirectoryName(Path.GetFullPath(executor)) + " & " + Path.GetFileName(executor) + " \"" + db + "\"";
                    // task.CommandLine = Path.GetFileName(executor) + " \"" + db + "\"";
                    task.IsExclusive  = false;
                    task.IsRerunnable = true;
                    task.Name         = "Worker";
                    task.Runtime      = taskTimeout;

                    hpcJob.AddTask(task);
                }

                scheduler.AddJob(hpcJob);
                scheduler.SubmitJob(hpcJob, null, null);
            }
            catch (Exception ex)
            {
                scheduler.CancelJob(hpcJob.Id, "Aborted.");
                throw ex;
            }
        }
Пример #6
0
        public void SubmitHPCRecoveryJob(string db, int eid, string cluster, int priority, int nworkers, string executor, string jobTemplate, int jobTimeout, int taskTimeout)
        {
            SqlConnection sql = Connect(db);
            SqlCommand    cmd = new SqlCommand("SELECT * FROM Experiments WHERE ID=" + eid, sql);
            SqlDataReader r   = cmd.ExecuteReader();

            if (!r.Read())
            {
                throw new Exception("Could not read from database.");
            }

            string nodegroup = (string)r["Nodegroup"];
            string locality  = (string)r["Locality"];
            string sharedDir = (string)r["SharedDir"];

            string eFullpath = System.IO.Path.GetFullPath(executor);
            string eFilename = System.IO.Path.GetFileName(executor);
            string sExecutor = eid.ToString() + "_" + eFilename;

            if (!File.Exists(sharedDir + "\\" + sExecutor) ||
                File.GetLastWriteTime(sharedDir + "\\" + sExecutor) < File.GetLastWriteTime(eFullpath))
            {
retry:
                try
                {
                    File.Copy(eFullpath, sharedDir + "\\" + sExecutor, true);
                }
                catch (UnauthorizedAccessException)
                {
                    sExecutor = sExecutor.Substring(0, sExecutor.Length - 4) + "-.exe";
                    goto retry;
                }
                catch (IOException)
                {
                    sExecutor = sExecutor.Substring(0, sExecutor.Length - 4) + "-.exe";
                    goto retry;
                }
            }

            scheduler.Connect(cluster);

            ISchedulerJob hpcJob = scheduler.CreateJob();

            if (jobTemplate != null)
            {
                hpcJob.SetJobTemplate(jobTemplate);
            }
            if (jobTimeout != 0)
            {
                hpcJob.Runtime = jobTimeout;
            }

            try
            {
                if (nodegroup != "<Any>")
                {
                    hpcJob.NodeGroups.Add(nodegroup);
                }
                hpcJob.Name        = "Z3 Performance Test Recovery (" + eid + ")";
                hpcJob.IsExclusive = true;
                hpcJob.CanPreempt  = true;
                SetPriority(hpcJob, priority);
                hpcJob.Project = "Z3";

                if (locality == "Socket")
                {
                    hpcJob.UnitType = JobUnitType.Socket;
                }
                else if (locality == "Core")
                {
                    hpcJob.UnitType = JobUnitType.Core;
                }
                else if (locality == "Node")
                {
                    hpcJob.UnitType = JobUnitType.Node;
                }
                else
                {
                    throw new Exception("Unknown locality.");
                }

                int max = nworkers == 0 ? 1 : nworkers;

                int progressTotal = max + 3;

                // Add population task.
                ReportProgress(Convert.ToInt32(100.0 * 1 / (double)max));
                ISchedulerTask populateTask = hpcJob.CreateTask();
                SetResources(populateTask, locality);
                populateTask.IsRerunnable  = false;
                populateTask.IsExclusive   = false;
                populateTask.WorkDirectory = sharedDir;
                populateTask.CommandLine   = sExecutor + " " + eid + " * \"" + db + "\""; // * means recovery
                populateTask.Name          = "Populate";
                if (taskTimeout != 0)
                {
                    populateTask.Runtime = taskTimeout;
                }
                populateTask.FailJobOnFailure = true;
                hpcJob.AddTask(populateTask);

                for (int i = 0; i < max; i++)
                {
                    // Add worker task.
                    ReportProgress(Convert.ToInt32(100.0 * (i + 1) / (double)max));
                    ISchedulerTask task = hpcJob.CreateTask();
                    SetResources(task, locality);
                    task.WorkDirectory = sharedDir;
                    task.CommandLine   = sExecutor + " " + eid + " \"" + db + "\"";
                    task.IsExclusive   = false;
                    task.IsRerunnable  = true;
                    task.DependsOn.Add("Populate");
                    task.Name             = "Worker";
                    task.FailJobOnFailure = false;
                    if (taskTimeout != 0)
                    {
                        task.Runtime = taskTimeout;
                    }
                    hpcJob.AddTask(task);
                }

                // No additional recovery task.

                // Add deletion task.
                ReportProgress(Convert.ToInt32(100.0 * (progressTotal) / (double)max));
                ISchedulerTask dTask = hpcJob.CreateTask();
                SetResources(dTask, locality);
                dTask.IsRerunnable  = true;
                dTask.IsExclusive   = false;
                dTask.WorkDirectory = sharedDir;
                dTask.CommandLine   = "del " + sharedDir + "\\" + executor;
                dTask.Name          = "Delete worker";
                dTask.DependsOn.Add("Worker");
                if (taskTimeout != 0)
                {
                    dTask.Runtime = taskTimeout;
                }
                dTask.FailJobOnFailure = false;
                hpcJob.AddTask(dTask);

                scheduler.AddJob(hpcJob);
                scheduler.SubmitJob(hpcJob, null, null);
            }
            catch (Exception ex)
            {
                MessageBox.Show("Error submitting job: " + ex.Message, "Error", MessageBoxButton.OK, MessageBoxImage.Error);
            }
        }