/// <summary>
/// Schedule a batch job
/// </summary>
/// <returns></returns>
private JobConfig ScheduleSingleJob(JobDeploymentSession job, string destFolder, JsonConfig defaultJobConfig, bool isOneTime, TimeSpan interval, TimeSpan window, DateTime processTime, DateTime scheduledTime, string prefix = "")
{
var ps_s = processTime;
var ps_e = processTime.Add(interval).AddMilliseconds(-1); //ENDTIME
var pe_s = ps_s.Add(-window);
var pe_e = ps_e.Add(-window); // STARTTIME
var dateString = ConvertDateToString(scheduledTime);
var suffix = prefix + $"-{Regex.Replace(dateString, "[^0-9]", "")}";
var jobName = job.Name + suffix;
job.SetStringToken("name", jobName);
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
var processStartTime = ConvertDateToString(pe_e);
var processEndTime = ConvertDateToString(ps_e);
destFolder = GetJobConfigFilePath(isOneTime, dateString, destFolder);
var jc = new JobConfig
{
Content = GetBatchConfigContent(job, defaultJobConfig.ToString(), processStartTime, processEndTime),
FilePath = ResourcePathUtil.Combine(destFolder, job.Name + ".conf"),
Name = jobName,
SparkJobName = job.SparkJobName + suffix,
ProcessStartTime = processStartTime,
ProcessEndTime = processEndTime,
ProcessingTime = dateString,
IsOneTime = isOneTime
};
return(jc);
}
/// <summary>
/// create or update the spark job entry for the given job in deploying
/// </summary>
/// <returns>name of the spark job deployed</returns>
private async Task <string[]> DeploySingleJob(JobDeploymentSession job, JsonConfig defaultSparkJobConfig)
{
List <string> names = new List <string>();
foreach (var jc in job.JobConfigs)
{
// For each job
// replace config with tokens from job and flow, and the runtime config file path
// create spark job entry
job.SparkJobName = jc.SparkJobName;
job.SparkJobConfigFilePath = jc.SparkFilePath;
var json = job.Tokens.Resolve(defaultSparkJobConfig);
var newJob = SparkJobConfig.From(json);
var jobName = newJob.Name;
var existingJob = await SparkJobData.GetByName(jobName);
if (existingJob != null)
{
//keep the state of the old job so we can stop that
newJob.SyncResult = existingJob.SyncResult;
}
var result = await this.SparkJobData.UpsertByName(jobName, newJob);
if (!result.IsSuccess)
{
throw new ConfigGenerationException($"Failed to upsert into SparkJob table for job '{jobName}': {result.Message}");
}
names.Add(jobName);
}
return(names.ToArray());
}
/// <summary>
/// Generate job config with the jobs scheduled
/// </summary>
/// <returns></returns>
private async Task GenerateJobConfigContent(JobDeploymentSession job, string destFolder, JsonConfig defaultJobConfig)
{
Ensure.NotNull(destFolder, "destFolder");
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
var inputConfig = job.Flow.Config?.GetGuiConfig();
var jcons = new List <JobConfig>();
for (int i = 0; i < inputConfig.BatchList.Length; i++)
{
job.JobConfigs.AddRange(await GetJobConfig(job, inputConfig.BatchList[i], destFolder, defaultJobConfig, i).ConfigureAwait(false));
}
}
private void GenerateJobConfigContent(JobDeploymentSession job, string destFolder, JsonConfig defaultJobConfig)
{
Ensure.NotNull(destFolder, "destFolder");
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
// replace config with tokens from job and the flow
var newJobConfig = job.Tokens.Resolve(defaultJobConfig);
Ensure.NotNull(newJobConfig, "newJobConfig");
job.SetJsonToken(TokenName_JobConfigContent, newJobConfig);
var destinationPath = ResourcePathUtil.Combine(destFolder, job.Name + ".json");
job.SetStringToken(TokenName_JobConfigFilePath, destinationPath);
}
/// <summary>
/// Get a batch job config
/// </summary>
/// <returns></returns>
private static string GetBatchConfigContent(JobDeploymentSession job, string content, string processStartTime, string processEndTime)
{
var specsBackup = job.GetAttachment <InputBatchingSpec[]>(TokenName_InputBatching);
foreach (var spec in specsBackup)
{
spec.ProcessStartTime = processStartTime;
spec.ProcessEndTime = processEndTime;
}
job.SetObjectToken(TokenName_InputBatching, specsBackup);
var jsonContent = job.Tokens.Resolve(content);
return(jsonContent);
}
private void GenerateJobConfigContent(JobDeploymentSession job, string destFolder, JsonConfig defaultJobConfig)
{
Ensure.NotNull(destFolder, "destFolder");
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
// replace config with tokens from job and the flow
var newJobConfig = job.Tokens.Resolve(defaultJobConfig);
Ensure.NotNull(newJobConfig, "newJobConfig");
job.SetJsonToken(TokenName_JobConfigContent, newJobConfig);
var jc = new JobConfig
{
Content = newJobConfig.ToString(),
FilePath = ResourcePathUtil.Combine(destFolder, job.Name + ".conf"),
SparkJobName = job.SparkJobName,
};
job.JobConfigs.Add(jc);
}
/// <summary>
/// Prepare for scheduling batch jobs
/// e.g. Calculate the processTime and startTime and endTime
/// </summary>
/// <returns></returns>
private async Task <List <JobConfig> > GetJobConfig(JobDeploymentSession job, FlowGuiInputBatchJob batchingJob,
string destFolder,
JsonConfig defaultJobConfig,
int index)
{
bool isOneTime = batchingJob.Type == Constants.Batch_OneTime;
var jQueue = new List <JobConfig>();
var batchProps = batchingJob.Properties;
if (batchingJob.Disabled || batchProps.StartTime == null || (isOneTime && batchProps.EndTime == null))
{
return(jQueue);
}
var configStartTime = (DateTime)batchProps.StartTime;
var configEndTime = batchProps.EndTime;
var interval = TranslateInterval(batchProps.Interval, batchProps.IntervalType);
var delay = TranslateDelay(batchProps.Delay, batchProps.DelayType);
var window = TranslateWindow(batchProps.Window, batchProps.WindowType);
var currentTime = DateTime.UtcNow;
if (!isOneTime)
{
if (currentTime < configStartTime || (configEndTime != null && configEndTime < currentTime))
{
await DisableBatchConfig(job.Flow.Config, index).ConfigureAwait(false);
return(jQueue);
}
}
DateTime startTime;
DateTime?endTime;
var prefix = "";
if (!isOneTime)
{
if (string.IsNullOrEmpty(batchProps.LastProcessedTime))
{
startTime = configStartTime;
endTime = currentTime;
}
else
{
var lastProcessedTimeFromConfig = UnixTimestampToDateTime(batchProps.LastProcessedTime);
var startTimeBasedOnLastProcessedTImeFromConfig = lastProcessedTimeFromConfig.Add(interval);
startTime = startTimeBasedOnLastProcessedTImeFromConfig;
endTime = currentTime;
}
}
else
{
prefix = "-OneTime";
startTime = configStartTime;
endTime = configEndTime;
}
DateTime lastProcessingTime = new DateTime();
for (var processingTime = startTime; processingTime <= endTime; processingTime += interval)
{
lastProcessingTime = processingTime;
var processingTimeBasedOnInterval = NormalizeTimeBasedOnInterval(processingTime, batchProps.IntervalType, new TimeSpan());
var processingTimeBasedOnDelay = NormalizeTimeBasedOnInterval(processingTime, batchProps.IntervalType, delay);
JobConfig jc = ScheduleSingleJob(job, destFolder, defaultJobConfig, isOneTime, interval, window, processingTimeBasedOnDelay, processingTimeBasedOnInterval, prefix);
jQueue.Add(jc);
}
if (!isOneTime)
{
if (lastProcessingTime != DateTime.MinValue)
{
var uTimestamp = DateTimeToUnixTimestamp(lastProcessingTime);
if (uTimestamp > 0)
{
var ret = await UpdateLastProcessedTime(job.Flow.Config, index, uTimestamp).ConfigureAwait(false);
if (!ret.IsSuccess)
{
throw new ConfigGenerationException(ret.Message);
}
}
}
}
else
{
// OneTime
var ret = await DisableBatchConfig(job.Flow.Config, index).ConfigureAwait(false);
if (!ret.IsSuccess)
{
throw new ConfigGenerationException(ret.Message);
}
}
return(jQueue);
}
