/// <summary>
        /// Job Management to stop a job by name if it is not stopped yet
        /// </summary>
        /// <param name="jobName">job name</param>
        /// <returns></returns>
        public async Task <Result> StopJob(string jobName)
        {
            Ensure.NotNull(jobName, "jobName");
            Logger.LogInformation($"stoping job '{jobName}'");

            var job = await SyncJobState(jobName);

            var jobState = job.SyncResult?.JobState;

            switch (jobState)
            {
            case JobState.Starting:
            case JobState.Running:
                var sparkJobClient = await ClusterManager.GetSparkJobClient(job.Cluster, job.DatabricksToken);

                var result = await sparkJobClient.StopJob(job.SyncResult.ClientCache);

                job.SyncResult = result;
                return(await this.JobData.UpdateSyncResultByName(jobName, result));

            case JobState.Success:
            case JobState.Idle:
                return(new SuccessResult($"job '{jobName}' has already been stopped"));

            case JobState.Error:
                return(new SuccessResult($"job '{jobName}' is currently in an error state"));

            default:
                throw new GeneralException($"Unexpected state '{jobState}' for job '{jobName}'");
            }
        }
        /// <summary>
        /// Job management to start a job by name
        /// </summary>
        /// <param name="jobName">internal job name</param>
        /// <returns></returns>
        public async Task<Result> StartJob(string jobName)
        {
            Ensure.NotNull(jobName, "jobName");
            Logger.LogInformation($"starting job '{jobName}'");
            var job = await EnsureJobState(jobName, JobState.Idle);
            var sparkJobClient = await ClusterManager.GetSparkJobClient(job.Cluster);
            var result = await sparkJobClient.SubmitJob(job.Options);

            // Update job state
            job.SyncResult = result;
            return await this.JobData.UpdateSyncResultByName(jobName, result);
        }
        /// <summary>
        /// Get job details for a specific job id from Job manager
        /// </summary>
        /// <param name="jobName">jobName</param>
        /// <returns></returns>
        public async Task <SparkJobConfig> SyncJobState(string jobName)
        {
            Ensure.NotNull(jobName, "jobName");
            Logger.LogInformation($"sync'ing job '{jobName}'");

            // Get the job config data from internal data and ensure job is in Idle state
            var job = await JobData.GetByName(jobName);

            Ensure.NotNull(job, "job");

            var state = job.SyncResult;

            if (state == null)
            {
                state = new SparkJobSyncResult();
            }

            if (state.ClientCache != null && state.ClientCache.Type != JTokenType.Null)
            {
                var sparkJobClient = await ClusterManager.GetSparkJobClient(job.Cluster, job.DatabricksToken);

                var newResult = await sparkJobClient.GetJobInfo(state.ClientCache);

                state = newResult;
                Logger.LogInformation($"got state for job '{jobName}'");
            }
            else
            {
                state.JobState = JobState.Idle;
                Logger.LogInformation($"no client cache for job '{jobName}', set state to '{state.JobState}'");
            }

            if (!state.Equals(job.SyncResult))
            {
                var updateResult = await JobData.UpdateSyncResultByName(jobName, state);

                if (!updateResult.IsSuccess)
                {
                    throw new GeneralException($"Failed to update job client cache for name '{jobName}': returned message '{updateResult.Message}'");
                }

                job.SyncResult = state;
                Logger.LogInformation($"done sync'ing the state for job '{jobName}', state updated");
            }
            else
            {
                Logger.LogInformation($"done sync'ing the state for job '{jobName}', no changes");
            }

            return(job);
        }