/// <summary>
/// create or update the spark job entry for the given job in deploying
/// </summary>
/// <returns>name of the spark job deployed</returns>
private async Task <string[]> DeploySingleJob(JobDeploymentSession job, JsonConfig defaultSparkJobConfig)
{
List <string> names = new List <string>();
foreach (var jc in job.JobConfigs)
{
// For each job
// replace config with tokens from job and flow, and the runtime config file path
// create spark job entry
job.SparkJobName = jc.SparkJobName;
job.SparkJobConfigFilePath = jc.SparkFilePath;
var json = job.Tokens.Resolve(defaultSparkJobConfig);
var newJob = SparkJobConfig.From(json);
var jobName = newJob.Name;
var existingJob = await SparkJobData.GetByName(jobName);
if (existingJob != null)
{
//keep the state of the old job so we can stop that
newJob.SyncResult = existingJob.SyncResult;
}
var result = await this.SparkJobData.UpsertByName(jobName, newJob);
if (!result.IsSuccess)
{
throw new ConfigGenerationException($"Failed to upsert into SparkJob table for job '{jobName}': {result.Message}");
}
names.Add(jobName);
}
return(names.ToArray());
}
public async Task <SparkJobConfig> GetByName(string jobName)
{
return(SparkJobConfig.From(await this.Storage.GetByName(jobName, DataCollectionName)));
}
public async Task <SparkJobConfig[]> GetAll()
{
var jsons = await this.Storage.GetAll(DataCollectionName);
return(jsons.Select(x => SparkJobConfig.From(x)).ToArray());
}
public async Task <SparkJobConfig[]> GetByNames(string[] names)
{
var jsons = await this.Storage.GetByNames(names, DataCollectionName);
return(jsons.Select(x => SparkJobConfig.From(x)).ToArray());
}
public Task <Result> UpsertByName(string jobName, SparkJobConfig sparkJobConfig)
{
return(Storage.SaveByName(jobName, sparkJobConfig.ToString(), DataCollectionName));
}
