public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
var schema = guiConfig.Input?.Properties?.InputSchemaFile;
Ensure.NotNull(schema, "guiConfig.input.properties.inputschemafile");
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "inputschema.json");
var schemaFile = await RuntimeStorage.SaveFile(filePath, schema);
var secretName = $"{config.Name}-inputschemafile";
var schemaFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, schemaFile, Configuration[Constants.ConfigSettingName_SparkType]);
flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
var projectColumns = guiConfig.Input?.Properties?.NormalizationSnippet?.Trim('\t', ' ', '\r', '\n');
//TODO: make the hardcoded "Raw.*" configurable?
var finalProjections = string.IsNullOrEmpty(projectColumns) ? "Raw.*" : projectColumns;
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "projection.txt");
var savedFile = await RuntimeStorage.SaveFile(filePath, finalProjections);
var secretName = $"{config.Name}-projectionfile";
var savedSecretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, savedFile, Configuration[Constants.ConfigSettingName_SparkType]);
flowToDeploy.SetObjectToken(TokenName_ProjectionFiles, new string[] { savedSecretId });
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
// Deploy job configs
var jobs = flowToDeploy.GetJobs();
var deploymentTasks = jobs?.Select(async job => {
var content = job.GetTokenString(GenerateJobConfig.TokenName_JobConfigContent);
var filePath = job.GetTokenString(GenerateJobConfig.TokenName_JobConfigFilePath);
if (content != null && filePath != null)
{
job.SparkJobConfigFilePath = await this.JobData.SaveFile(filePath, content);
}
else
{
job.SparkJobConfigFilePath = null;
}
return(job.SparkJobConfigFilePath);
});
// Ensure all jobs configs are written successfully
if (deploymentTasks == null)
{
return("no jobs, skipped");
}
await Task.WhenAll(deploymentTasks);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped");
}
var referenceData = guiConfig?.Input?.ReferenceData ?? Array.Empty <FlowGuiReferenceData>();
var specs = referenceData.Select(rd =>
{
var props = rd.Properties;
Ensure.NotNull(props, $"guiConfig.input.referenceData['{rd.Id}'].properties");
Ensure.NotNull(rd.Type, $"guiConfig.input.referenceData['{rd.Id}'].type");
Ensure.NotNull(props.Path, $"guiConfig.input.referenceData['{rd.Id}'].properties.path");
return(new ReferenceDataSpec()
{
Name = rd.Id,
Path = props.Path,
Format = rd.Type,
Delimiter = props.Delimiter,
Header = props.Header.ToString()
});
}).ToArray();
flowToDeploy.SetObjectToken(TokenName_ReferenceData, specs);
await Task.Yield();
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
// get a flattener
var flattener = await this.ConfigFlatteners.GetDefault();
if (flattener == null)
{
return("no flattern config, skipped");
}
// flatten each job config
var jobs = flowToDeploy.GetJobs();
if (jobs == null)
{
return("no jobs, skipped");
}
foreach (var job in jobs)
{
foreach (var jc in job.JobConfigs)
{
var jsonContent = job.Tokens.Resolve(jc.Content);
if (jsonContent != null)
{
var json = JsonConfig.From(jsonContent);
jc.Content = flattener.Flatten(json);
}
}
}
return("done");
}
/// <summary>
/// Generate and set the info for the projection file which will be used to generate JobConfig
/// </summary>
/// <returns></returns>
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
if (runtimeKeyVaultName != "local")
{
var secretName = $"{config.Name}-projectionfile";
Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType);
var projectionFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
flowToDeploy.SetObjectToken(TokenName_ProjectionFiles, new string[] { projectionFileSecret });
}
else
{
flowToDeploy.SetObjectToken(TokenName_ProjectionFiles, new string[] { flowToDeploy.ResultProperties[PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder].ToString() + "/projection.txt" });
}
await Task.CompletedTask;
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject);
Ensure.NotNull(rulesCode, "rulesCode");
Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot");
Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics");
var stateTables = rulesCode.AccumlationTables.Select(t =>
{
return(new StateTableSpec()
{
Name = t.Key,
Schema = t.Value,
Location = ConstructStateTablePath(guiConfig.Name, t.Key)
});
}).ToArray();
flowToDeploy.SetObjectToken(TokenName_StateTables, stateTables);
await Task.Yield();
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
var rulesCode = flowToDeploy.GetAttachment <RulesCode>(GenerateTransformFile.AttachmentName_CodeGenObject);
Ensure.NotNull(rulesCode, "rulesCode");
Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot");
Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics");
var timewindows = rulesCode.TimeWindows.Select(t =>
{
return(new TimeWindowSpec()
{
Name = t.Key,
WindowDuration = t.Value
});
}).ToArray();
flowToDeploy.SetObjectToken(TokenName_TimeWindows, timewindows);
await Task.Yield();
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
if (flowConfig.GetGuiConfig()?.Input?.Mode != Constants.InputMode_Batching)
{
return("done");
}
// set the default job config
var defaultJobConfig = JsonConfig.From(flowConfig.CommonProcessor?.Template);
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
flowToDeploy.SetAttachment(TokenName__DefaultJobConfig, defaultJobConfig);
// Deploy job configs
var jobsToDeploy = flowToDeploy?.GetJobs();
if (jobsToDeploy != null)
{
foreach (var job in jobsToDeploy)
{
await GenerateJobConfigContent(job, job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder), defaultJobConfig).ConfigureAwait(false);
}
return("done");
}
else
{
await Task.Yield();
return("no jobs, skipped");
}
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
//TODO: create eventhub consumer group
var flowConfig = flowToDeploy.Config;
var config = flowConfig?.GetGuiConfig()?.Input;
//TODO: take care of other input types
if (config != null && "iothub".Equals(config.InputType, StringComparison.InvariantCultureIgnoreCase))
{
var props = config.Properties;
if (props != null)
{
var connectionString = props.InputEventhubConnection;
flowToDeploy.SetStringToken(TokenName_InputEventHubConnectionString, connectionString);
// TODO: figure out the right value
flowToDeploy.SetStringToken(TokenName_InputEventHubConsumerGroup, flowConfig.Name);
flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointDir, $"hdfs://mycluster/dataxdirect/{JobMetadata.TokenPlaceHolder_JobName}/eventhub/checkpoints");
var intervalInSeconds = props?.WindowDuration;
flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointInterval, intervalInSeconds);
flowToDeploy.SetObjectToken(TokenName_InputEventHubMaxRate, props.MaxRate);
flowToDeploy.SetObjectToken(TokenName_InputEventHubFlushExistingCheckpoints, true);
}
}
await Task.Yield();
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
// set the default job config
var defaultJobConfig = JsonConfig.From(flowConfig.CommonProcessor?.Template);
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
flowToDeploy.SetAttachment(ParameterObjectName_DefaultJobConfig, defaultJobConfig);
// Deploy job configs
var jobsToDeploy = flowToDeploy?.GetJobs();
if (jobsToDeploy != null)
{
foreach (var job in jobsToDeploy)
{
GenerateJobConfigContent(job, job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder), defaultJobConfig);
}
return("done");
}
else
{
await Task.Yield();
return("no jobs, skipped");
}
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
if (flowToDeploy.Config.GetGuiConfig().Input?.Mode == Constants.InputMode_Batching)
{
var inputConfig = flowToDeploy.Config.GetGuiConfig();
var inputBatching = inputConfig.Input.Batch ?? Array.Empty <FlowGuiInputBatchInput>();
var specsTasks = inputBatching.Select(async rd =>
{
var connectionString = await KeyVaultClient.ResolveSecretUriAsync(rd.Properties.Connection).ConfigureAwait(false);
var inputPath = await KeyVaultClient.ResolveSecretUriAsync(rd.Properties.Path).ConfigureAwait(false);
return(new InputBatchingSpec()
{
Name = ConfigHelper.ParseBlobAccountName(connectionString),
Path = rd.Properties.Path,
Format = rd.Properties.FormatType,
CompressionType = rd.Properties.CompressionType,
ProcessStartTime = "",
ProcessEndTime = "",
PartitionIncrement = GetPartitionIncrement(inputPath).ToString(CultureInfo.InvariantCulture),
});
}).ToArray();
var specs = await Task.WhenAll(specsTasks).ConfigureAwait(false);
flowToDeploy.SetAttachment(TokenName_InputBatching, specs);
}
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var jobs = flowToDeploy.GetJobs();
if (jobs == null || !jobs.Where(j => j.JobConfigs.Any()).Any())
{
return("no jobs, skipped");
}
var config = flowToDeploy.Config;
var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject);
Ensure.NotNull(rulesCode, "rulesCode");
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt");
var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code);
var transformFileSecret = flowToDeploy.GetTokenString(PrepareTransformFile.TokenName_TransformFile);
await KeyVaultClient.SaveSecretAsync(transformFileSecret, transformFilePath);
return("done");
}
/// <summary>
/// Generate and set the info for the transform file which will be used to generate JobConfig
/// </summary>
/// <returns></returns>
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
string queries = string.Join("\n", guiConfig.Process?.Queries);
string ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name);
RulesCode rulesCode = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name);
Ensure.NotNull(rulesCode, "rulesCode");
// Save the rulesCode object for downstream processing
flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode);
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var secretName = $"{config.Name}-transform";
Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType);
var transformFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);
await Task.CompletedTask;
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
await Task.Yield();
// Go through a pass of validation to ensure required fields are present
var result = this.ValidateFlowConfig(flowToDeploy.Config);
return($"result is {result.IsSuccess}");
}
/// <summary>
/// Delete the runtime job configs
/// </summary>
/// <param name="flowToDelete"></param>
/// <returns></returns>
public override async Task <string> Delete(FlowDeploymentSession flowToDelete)
{
var flowConfig = flowToDelete.Config;
var runtimeConfigsFolder = flowConfig.GetJobConfigDestinationFolder();
flowToDelete.SetStringToken(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder, runtimeConfigsFolder);
var folderToDelete = flowToDelete.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
return(await this.JobData.DeleteConfigs(folderToDelete));
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var guiConfig = flowToDeploy.Config.GetGuiConfig();
if (guiConfig?.Input?.Mode == Constants.InputMode_Batching)
{
var jobConfigs = flowToDeploy.GetJobs().First().JobConfigs;
var reConfigs = jobConfigs.Where(j => !j.IsOneTime).ToList();
var otConfigs = jobConfigs.Where(j => j.IsOneTime).ToList();
Assert.AreEqual(expected: 2, actual: reConfigs.Count);
Assert.AreEqual(expected: 3, actual: otConfigs.Count);
var startTimeConfig = (DateTime)guiConfig.BatchList[0].Properties.StartTime;
var curTime = startTimeConfig.AddDays(1);
var normalizedStartTime = startTimeConfig.Add(-startTimeConfig.TimeOfDay);
TimeSpan interval = new TimeSpan(1, 0, 0, 0);
var expected = normalizedStartTime;
foreach (var c in reConfigs)
{
var actualStart = DateTime.Parse(c.ProcessStartTime).ToUniversalTime();
var actualEnd = DateTime.Parse(c.ProcessEndTime).ToUniversalTime();
var window = (actualEnd - actualStart).TotalSeconds;
Assert.AreEqual(expected: expected.AddDays(-2), actual: actualStart, message: $"StartTime:{actualStart}");
Assert.AreEqual(expected: expected.AddDays(1).AddSeconds(-1), actual: actualEnd, message: $"EndTime:{actualEnd}");
Assert.AreEqual(expected: 259199, actual: window, message: $"Window:{window}");
Assert.IsTrue(actualStart < curTime);
expected = expected.Add(interval);
}
var lastScheduleTime = DateTime.Parse(reConfigs.Last().ProcessStartTime).ToUniversalTime();
Assert.IsTrue(lastScheduleTime < curTime);
expected = normalizedStartTime;
foreach (var c in otConfigs)
{
var actualStart = DateTime.Parse(c.ProcessStartTime).ToUniversalTime();
var actualEnd = DateTime.Parse(c.ProcessEndTime).ToUniversalTime();
var window = (actualEnd - actualStart).TotalSeconds;
Assert.AreEqual(expected: expected, actual: actualStart, message: $"StartTime:{actualStart}");
Assert.AreEqual(expected: expected.AddDays(1).AddSeconds(-1), actual: actualEnd, message: $"EndTime:{actualEnd}");
Assert.AreEqual(expected: 86399, actual: window, message: $"Window:{window}");
expected = expected.Add(interval);
}
lastScheduleTime = DateTime.Parse(otConfigs.Last().ProcessStartTime).ToUniversalTime();
Assert.IsTrue(lastScheduleTime > curTime);
}
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var expectedConfigContent = await File.ReadAllTextAsync(@"Resource\jobConfig.json");
var expectedJson = JsonConfig.From(expectedConfigContent);
var actualContentContent = flowToDeploy.GetJobs().First().GetTokenString(GenerateJobConfig.TokenName_JobConfigContent);
var actualJson = JsonConfig.From(actualContentContent);
foreach (var match in JsonConfig.Match(expectedJson, actualJson))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config.GetGuiConfig();
flowToDeploy.SetStringToken(TokenName_InputStreamingCheckpointDir, $"hdfs://mycluster/dataxdirect/{JobMetadata.TokenPlaceHolder_JobName}/streaming/checkpoints");
var intervalInSeconds = guiConfig?.Input?.Properties?.WindowDuration;
flowToDeploy.SetStringToken(TokenName_InputStreamingInterval, intervalInSeconds);
await Task.Yield();
return("done");
}
public override Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var functions = config.GetGuiConfig()?.Process?.Functions;
if (functions == null)
{
return(Task.FromResult("no functions defined in gui input, skipped."));
}
var azureFuncs = GetAzureFuncDefinitions(functions);
flowToDeploy.SetObjectToken(TokenName_AzureFunctions, azureFuncs);
return(Task.FromResult("done"));
}
/// <summary>
/// Delete the job config
/// </summary>
/// <param name="flowToDelete"></param>
/// <returns></returns>
public override async Task <string> Delete(FlowDeploymentSession flowToDelete)
{
var jobNames = flowToDelete?.Config?.JobNames;
if (jobNames == null)
{
return("skipped");
}
foreach (var jobName in jobNames)
{
await SparkJobData.DeleteByName(jobName);
}
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
flowToDeploy.SetStringToken(TokenName_ProcessTimestampColumn, guiConfig.Process.TimestampColumn);
flowToDeploy.SetStringToken(TokenName_ProcessWatermark, guiConfig.Process.Watermark);
await Task.Yield();
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config.GetGuiConfig();
var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;
var inputStreamingCheckpointDirPrefix = (sparkType == Constants.SparkTypeDataBricks) ? Constants.PrefixDbfs : Constants.PrefixHdfs;
flowToDeploy.SetStringToken(TokenName_InputStreamingCheckpointDir, $"{inputStreamingCheckpointDirPrefix}mycluster/dataxdirect/{JobMetadata.TokenPlaceHolder_JobName}/streaming/checkpoints");
var intervalInSeconds = guiConfig?.Input?.Properties?.WindowDuration;
flowToDeploy.SetStringToken(TokenName_InputStreamingInterval, intervalInSeconds);
await Task.Yield();
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
// determine the runtime config folder
var version = VersionGeneration.Next();
flowToDeploy.SetStringToken(TokenName_ConfigVersion, version);
var runtimeConfigFolder = this.JobData.FigureOutDestinationFolder(flowConfig.GetJobConfigDestinationFolder(), version);
flowToDeploy.SetStringToken(TokenName_RuntimeConfigFolder, runtimeConfigFolder);
flowToDeploy.ResultProperties[ResultPropertyName_RuntimeConfigFolder] = flowToDeploy.GetTokenString(TokenName_RuntimeConfigFolder);
await Task.Yield();
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
var defaultConfig = await FlowData.GetFlowConfigByInputType(flowConfig.GetGuiConfig().Input.InputType, flowToDeploy.Tokens);
if (defaultConfig == null)
{
return("defaultConfig is null, skipped");
}
var newConfig = flowConfig.RebaseOn(defaultConfig);
flowToDeploy.UpdateFlowConfig(newConfig);
flowToDeploy.SetAttachment(AttachementName_DefaultFlowConfig, defaultConfig);
return("done");
}
private void PortConfigurationSetting(FlowDeploymentSession session, string settingName, bool isRequired = false)
{
if (Configuration.TryGet(settingName, out var value))
{
session.SetStringToken(settingName, value);
}
else
{
if (isRequired)
{
throw new ConfigGenerationException($"Required setting '{settingName}' is not found in global configuration");
}
else
{
session.SetNullToken(settingName);
}
}
}
/// <summary>
/// Generate and set the info for the input schema file which will be used to generate JobConfig
/// </summary>
/// <returns></returns>
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var secretName = $"{config.Name}-inputschemafile";
Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType);
var schemaFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret);
await Task.CompletedTask;
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var guiConfig = flowToDeploy.Config?.GetGuiConfig();
if (guiConfig == null)
{
// If guiConfig is empty, get the number of executors from job common token and convert it to integer
var executorsString = flowToDeploy.Config?.CommonProcessor?.JobCommonTokens?.GetOrDefault("sparkJobNumExecutors", null);
if (executorsString != null)
{
if (!int.TryParse(executorsString, out int executorsInt))
{
throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{executorsString}'.");
}
flowToDeploy.SetObjectToken("sparkJobNumExecutorsInt", executorsInt);
}
return("no gui config, skipped");
}
// Setting TokenName_SparkJobNumExecutors
var numExecutorsString = guiConfig?.Process?.JobConfig?.JobNumExecutors;
if (!int.TryParse(numExecutorsString, out int numExecutors))
{
throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{numExecutorsString}'.");
}
flowToDeploy.SetObjectToken(TokenName_SparkJobNumExecutors, numExecutors);
// Setting TokenName_SparkJobJobExecutorMemory
var jobExecutorMemoryString = guiConfig?.Process?.JobConfig?.JobExecutorMemory;
if (!int.TryParse(jobExecutorMemoryString, out int jobExecutorMemory))
{
throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobExecutorMemory:'{jobExecutorMemoryString}'.");
}
flowToDeploy.SetStringToken(TokenName_SparkJobJobExecutorMemory, $"{jobExecutorMemory}m");
await Task.Yield();
return("done");
}
private async Task <string> PortSettings(FlowDeploymentSession flowToDeploy)
{
// TODO: currently this is a whitelist only port a portion of configuration settings
// but we might want to port all configuration settings into the token dictionary
PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_RuntimeApplicationInsightKey, false);
PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_ClusterName, true);
PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_ServiceKeyVaultName, true);
PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_RuntimeKeyVaultName, true);
PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_ConfigFolderHost, false);
PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_ConfigFolderContainerPath, false);
PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_MetricEventHubConnectionKey, true);
PortConfigurationSetting(flowToDeploy, "eventHubResourceGroupName", false);
PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_LocalRoot, false);
PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_LocalMetricsHttpEndpoint, false);
await Task.Yield();
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var jobs = flowToDeploy.GetJobs();
if (jobs == null || !jobs.Where(j => j.JobConfigs.Any()).Any())
{
return("no jobs, skipped");
}
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
var projectColumns = guiConfig.Input?.Properties?.NormalizationSnippet?.Trim('\t', ' ', '\r', '\n');
//TODO: make the hardcoded "Raw.*" configurable?
var finalProjections = string.IsNullOrEmpty(projectColumns) ? "Raw.*" : projectColumns;
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "projection.txt");
var savedFile = await RuntimeStorage.SaveFile(filePath, finalProjections);
var tokenValue = flowToDeploy.GetTokenString(PrepareProjectionFile.TokenName_ProjectionFiles);
var projectionFileSecret = JArray.Parse(tokenValue).FirstOrDefault()?.Value <string>();
if (!string.IsNullOrEmpty(projectionFileSecret))
{
await KeyVaultClient.SaveSecretAsync(projectionFileSecret, savedFile);
}
return("done");
}
