Example #1
0
        public async Task ProcessJobAsync(DataModel.Job job)
        {
            switch (job.JobStatus)
            {
            case JobStatus.Submitted:
                return;

            case JobStatus.Pending:
                try
                {
                    var clusters = await FindClustersForJobAsync(job);

                    if (clusters == null || clusters.Count == 0)
                    {
                        Console.WriteLine("no cluster found for submitting the job");
                        job.LastProcessedTime = DateTime.Now;
                        await dataStore.UpdateJobAsync(job);

                        return;
                    }
                    Random r       = new Random();
                    int    index   = r.Next(0, clusters.Count);
                    var    cluster = clusters[index];
                    //var sparkSubmitTask = new SparkSubmitTask()
                    //{
                    //    Parameters = job.SparkSubmitTaskParams
                    //};
                    var jobSettings = JobSettings
                                      .GetNewSparkJarJobSettings(job.RefId, "class1", job.SparkSubmitTaskParams, job.SparkSubmitTaskParams)
                                      .WithExistingCluster(cluster.SparkClusterId);
                    //var jobSettings = new JobSettings()
                    //{
                    //    SparkSubmitTask = sparkSubmitTask,
                    //    ExistingClusterId = cluster.SparkClusterId
                    //};
                    var jobId = await databricksClient.Jobs.Create(jobSettings);

                    var runId = (await databricksClient.Jobs.RunNow(jobId, null)).RunId;
                    job.ClusterId = cluster.SparkClusterId;
                    job.JobId     = jobId.ToString();
                    job.RunId     = runId.ToString();
                    job.JobStatus = JobStatus.Submitted;
                } catch (Exception ex)
                {
                    Console.WriteLine("error while submitting spark job: " + ex.Message);
                    job.LastProcessedTime = DateTime.Now;
                    await dataStore.UpdateJobAsync(job);

                    return;
                }
                await dataStore.UpdateJobAsync(job);

                return;
            }
        }
Example #2
0
        private async Task <List <Cluster> > FindClustersForJobAsync(DataModel.Job job)
        {
            var clusterSearchParams = new ClusterSearchParams
            {
                JobPriority = job.JobPriority,
                JobType     = job.JobType,
                TenantId    = job.TenantId
            };

            return(await dataStore.SearchClustersAsync(clusterSearchParams));
        }
Example #3
0
        public async Task <SubmitJobResponse> SubmitJobAsync(SubmitJobRequest req)
        {
            if (!Enum.TryParse <JobType>(req.JobCriteria.JobType, out JobType jobType))
            {
                throw new SjaasServiceValidationException("invalid job type specified: " + req.JobCriteria.JobType);
            }
            var job = new DataModel.Job
            {
                RefId                 = Guid.NewGuid().ToString(),
                JobPriority           = req.JobCriteria.JobPriority,
                JobType               = jobType,
                TenantId              = req.TenantId,
                SparkSubmitTaskParams = req.SparkSubmitTaskParams,
                JobStatus             = JobStatus.Pending,
                LastProcessedTime     = DateTime.Now
            };
            await datastore.AddJobAsync(job);

            return(new SubmitJobResponse(job.RefId));
        }
Example #4
0
 /// <summary>
 /// Creates a job with specified name and sets it as current.
 /// </summary>
 /// <param name="name">
 /// Job's name.
 /// </param>
 /// <returns>
 /// An instance of <see cref="DataModel.Job"/>.
 /// </returns>
 public Dto.Job CreateJob(string name)
 {
     currentJob = scheduler.CreateJob(name);
     return currentJob.ToContract();
 }
Example #5
0
 /// <summary>
 /// Opens an existing job and sets it as current.
 /// </summary>
 /// <param name="id">
 /// Job's UID.
 /// </param>
 public void OpenJob(Guid id)
 {
     var job = scheduler.Jobs.Where(j => j.Id == id).Single();
     currentJob = job;
 }
Example #6
0
 /// <summary>
 /// Deletes current job from scheduler. Job must not be in state "Processing".
 /// </summary>
 public void DeleteJob()
 {
     scheduler.DeleteJob(currentJob);
     currentJob = null;
 }