/// <summary>
/// Get job details for a specific job id from Job manager
/// </summary>
/// <param name="jobName">jobName</param>
/// <returns></returns>
public async Task <SparkJobConfig> SyncJobState(string jobName)
{
Ensure.NotNull(jobName, "jobName");
Logger.LogInformation($"sync'ing job '{jobName}'");
// Get the job config data from internal data and ensure job is in Idle state
var job = await JobData.GetByName(jobName);
Ensure.NotNull(job, "job");
var state = job.SyncResult;
if (state == null)
{
state = new SparkJobSyncResult();
}
if (state.ClientCache != null && state.ClientCache.Type != JTokenType.Null)
{
var sparkJobClient = await ClusterManager.GetSparkJobClient(job.Cluster, job.DatabricksToken);
var newResult = await sparkJobClient.GetJobInfo(state.ClientCache);
state = newResult;
Logger.LogInformation($"got state for job '{jobName}'");
}
else
{
state.JobState = JobState.Idle;
Logger.LogInformation($"no client cache for job '{jobName}', set state to '{state.JobState}'");
}
if (!state.Equals(job.SyncResult))
{
var updateResult = await JobData.UpdateSyncResultByName(jobName, state);
if (!updateResult.IsSuccess)
{
throw new GeneralException($"Failed to update job client cache for name '{jobName}': returned message '{updateResult.Message}'");
}
job.SyncResult = state;
Logger.LogInformation($"done sync'ing the state for job '{jobName}', state updated");
}
else
{
Logger.LogInformation($"done sync'ing the state for job '{jobName}', no changes");
}
return(job);
}