public async Task ProcessJobAsync(DataModel.Job job)
{
switch (job.JobStatus)
{
case JobStatus.Submitted:
return;
case JobStatus.Pending:
try
{
var clusters = await FindClustersForJobAsync(job);
if (clusters == null || clusters.Count == 0)
{
Console.WriteLine("no cluster found for submitting the job");
job.LastProcessedTime = DateTime.Now;
await dataStore.UpdateJobAsync(job);
return;
}
Random r = new Random();
int index = r.Next(0, clusters.Count);
var cluster = clusters[index];
//var sparkSubmitTask = new SparkSubmitTask()
//{
// Parameters = job.SparkSubmitTaskParams
//};
var jobSettings = JobSettings
.GetNewSparkJarJobSettings(job.RefId, "class1", job.SparkSubmitTaskParams, job.SparkSubmitTaskParams)
.WithExistingCluster(cluster.SparkClusterId);
//var jobSettings = new JobSettings()
//{
// SparkSubmitTask = sparkSubmitTask,
// ExistingClusterId = cluster.SparkClusterId
//};
var jobId = await databricksClient.Jobs.Create(jobSettings);
var runId = (await databricksClient.Jobs.RunNow(jobId, null)).RunId;
job.ClusterId = cluster.SparkClusterId;
job.JobId = jobId.ToString();
job.RunId = runId.ToString();
job.JobStatus = JobStatus.Submitted;
} catch (Exception ex)
{
Console.WriteLine("error while submitting spark job: " + ex.Message);
job.LastProcessedTime = DateTime.Now;
await dataStore.UpdateJobAsync(job);
return;
}
await dataStore.UpdateJobAsync(job);
return;
}
}
private async Task <List <Cluster> > FindClustersForJobAsync(DataModel.Job job)
{
var clusterSearchParams = new ClusterSearchParams
{
JobPriority = job.JobPriority,
JobType = job.JobType,
TenantId = job.TenantId
};
return(await dataStore.SearchClustersAsync(clusterSearchParams));
}
public async Task <SubmitJobResponse> SubmitJobAsync(SubmitJobRequest req)
{
if (!Enum.TryParse <JobType>(req.JobCriteria.JobType, out JobType jobType))
{
throw new SjaasServiceValidationException("invalid job type specified: " + req.JobCriteria.JobType);
}
var job = new DataModel.Job
{
RefId = Guid.NewGuid().ToString(),
JobPriority = req.JobCriteria.JobPriority,
JobType = jobType,
TenantId = req.TenantId,
SparkSubmitTaskParams = req.SparkSubmitTaskParams,
JobStatus = JobStatus.Pending,
LastProcessedTime = DateTime.Now
};
await datastore.AddJobAsync(job);
return(new SubmitJobResponse(job.RefId));
}
/// <summary>
/// Creates a job with specified name and sets it as current.
/// </summary>
/// <param name="name">
/// Job's name.
/// </param>
/// <returns>
/// An instance of <see cref="DataModel.Job"/>.
/// </returns>
public Dto.Job CreateJob(string name)
{
currentJob = scheduler.CreateJob(name);
return currentJob.ToContract();
}
/// <summary>
/// Opens an existing job and sets it as current.
/// </summary>
/// <param name="id">
/// Job's UID.
/// </param>
public void OpenJob(Guid id)
{
var job = scheduler.Jobs.Where(j => j.Id == id).Single();
currentJob = job;
}
/// <summary>
/// Deletes current job from scheduler. Job must not be in state "Processing".
/// </summary>
public void DeleteJob()
{
scheduler.DeleteJob(currentJob);
currentJob = null;
}