/// <summary>
        /// Get job details for a specific job id from Job manager
        /// </summary>
        /// <param name="jobName">jobName</param>
        /// <returns></returns>
        public async Task <SparkJobConfig> SyncJobState(string jobName)
        {
            Ensure.NotNull(jobName, "jobName");
            Logger.LogInformation($"sync'ing job '{jobName}'");

            // Get the job config data from internal data and ensure job is in Idle state
            var job = await JobData.GetByName(jobName);

            Ensure.NotNull(job, "job");

            var state = job.SyncResult;

            if (state == null)
            {
                state = new SparkJobSyncResult();
            }

            if (state.ClientCache != null && state.ClientCache.Type != JTokenType.Null)
            {
                var sparkJobClient = await ClusterManager.GetSparkJobClient(job.Cluster, job.DatabricksToken);

                var newResult = await sparkJobClient.GetJobInfo(state.ClientCache);

                state = newResult;
                Logger.LogInformation($"got state for job '{jobName}'");
            }
            else
            {
                state.JobState = JobState.Idle;
                Logger.LogInformation($"no client cache for job '{jobName}', set state to '{state.JobState}'");
            }

            if (!state.Equals(job.SyncResult))
            {
                var updateResult = await JobData.UpdateSyncResultByName(jobName, state);

                if (!updateResult.IsSuccess)
                {
                    throw new GeneralException($"Failed to update job client cache for name '{jobName}': returned message '{updateResult.Message}'");
                }

                job.SyncResult = state;
                Logger.LogInformation($"done sync'ing the state for job '{jobName}', state updated");
            }
            else
            {
                Logger.LogInformation($"done sync'ing the state for job '{jobName}', no changes");
            }

            return(job);
        }
Example #2
0
 public Task <Result> UpdateSyncResultByName(string jobName, SparkJobSyncResult result)
 {
     return(UpdateObjectFieldByName(jobName, SparkJobConfig.FieldName_SyncResult, result));
 }