/// <summary> /// create or update the spark job entry for the given job in deploying /// </summary> /// <returns>name of the spark job deployed</returns> private async Task <string[]> DeploySingleJob(JobDeploymentSession job, JsonConfig defaultSparkJobConfig) { List <string> names = new List <string>(); foreach (var jc in job.JobConfigs) { // For each job // replace config with tokens from job and flow, and the runtime config file path // create spark job entry job.SparkJobName = jc.SparkJobName; job.SparkJobConfigFilePath = jc.SparkFilePath; var json = job.Tokens.Resolve(defaultSparkJobConfig); var newJob = SparkJobConfig.From(json); var jobName = newJob.Name; var existingJob = await SparkJobData.GetByName(jobName); if (existingJob != null) { //keep the state of the old job so we can stop that newJob.SyncResult = existingJob.SyncResult; } var result = await this.SparkJobData.UpsertByName(jobName, newJob); if (!result.IsSuccess) { throw new ConfigGenerationException($"Failed to upsert into SparkJob table for job '{jobName}': {result.Message}"); } names.Add(jobName); } return(names.ToArray()); }
public async Task <SparkJobConfig[]> GetByNames(string[] names) { var jsons = await this.Storage.GetByNames(names, DataCollectionName); return(jsons.Select(x => SparkJobConfig.From(x)).ToArray()); }
public async Task <SparkJobConfig> GetByName(string jobName) { return(SparkJobConfig.From(await this.Storage.GetByName(jobName, DataCollectionName))); }
public async Task <SparkJobConfig[]> GetAll() { var jsons = await this.Storage.GetAll(DataCollectionName); return(jsons.Select(x => SparkJobConfig.From(x)).ToArray()); }