public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
//TODO: create eventhub consumer group
var flowConfig = flowToDeploy.Config;
var config = flowConfig?.GetGuiConfig()?.Input;
//TODO: take care of other input types
if (config != null && "iothub".Equals(config.InputType, StringComparison.InvariantCultureIgnoreCase))
{
var props = config.Properties;
if (props != null)
{
var connectionString = props.InputEventhubConnection;
flowToDeploy.SetStringToken(TokenName_InputEventHubConnectionString, connectionString);
// TODO: figure out the right value
flowToDeploy.SetStringToken(TokenName_InputEventHubConsumerGroup, flowConfig.Name);
flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointDir, $"hdfs://mycluster/dataxdirect/{JobMetadata.TokenPlaceHolder_JobName}/eventhub/checkpoints");
var intervalInSeconds = props?.WindowDuration;
flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointInterval, intervalInSeconds);
flowToDeploy.SetObjectToken(TokenName_InputEventHubMaxRate, props.MaxRate);
flowToDeploy.SetObjectToken(TokenName_InputEventHubFlushExistingCheckpoints, true);
}
}
await Task.Yield();
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config.GetGuiConfig();
flowToDeploy.SetStringToken(TokenName_InputStreamingCheckpointDir, $"hdfs://mycluster/dataxdirect/{JobMetadata.TokenPlaceHolder_JobName}/streaming/checkpoints");
var intervalInSeconds = guiConfig?.Input?.Properties?.WindowDuration;
flowToDeploy.SetStringToken(TokenName_InputStreamingInterval, intervalInSeconds);
await Task.Yield();
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
var schema = guiConfig.Input?.Properties?.InputSchemaFile;
Ensure.NotNull(schema, "guiConfig.input.properties.inputschemafile");
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "inputschema.json");
var schemaFile = await RuntimeStorage.SaveFile(filePath, schema);
var secretName = $"{config.Name}-inputschemafile";
var schemaFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, schemaFile, Configuration[Constants.ConfigSettingName_SparkType]);
flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret);
return("done");
}
/// <summary>
/// Generate and set the info for the transform file which will be used to generate JobConfig
/// </summary>
/// <returns></returns>
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
string queries = string.Join("\n", guiConfig.Process?.Queries);
string ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name);
RulesCode rulesCode = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name);
Ensure.NotNull(rulesCode, "rulesCode");
// Save the rulesCode object for downstream processing
flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode);
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var secretName = $"{config.Name}-transform";
Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType);
var transformFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);
await Task.CompletedTask;
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config.GetGuiConfig();
var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;
var inputStreamingCheckpointDirPrefix = (sparkType == Constants.SparkTypeDataBricks) ? Constants.PrefixDbfs : Constants.PrefixHdfs;
flowToDeploy.SetStringToken(TokenName_InputStreamingCheckpointDir, $"{inputStreamingCheckpointDirPrefix}mycluster/dataxdirect/{JobMetadata.TokenPlaceHolder_JobName}/streaming/checkpoints");
var intervalInSeconds = guiConfig?.Input?.Properties?.WindowDuration;
flowToDeploy.SetStringToken(TokenName_InputStreamingInterval, intervalInSeconds);
await Task.Yield();
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
flowToDeploy.SetStringToken(TokenName_ProcessTimestampColumn, guiConfig.Process.TimestampColumn);
flowToDeploy.SetStringToken(TokenName_ProcessWatermark, guiConfig.Process.Watermark);
await Task.Yield();
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
// determine the runtime config folder
var version = VersionGeneration.Next();
flowToDeploy.SetStringToken(TokenName_ConfigVersion, version);
var runtimeConfigFolder = this.JobData.FigureOutDestinationFolder(flowConfig.GetJobConfigDestinationFolder(), version);
flowToDeploy.SetStringToken(TokenName_RuntimeConfigFolder, runtimeConfigFolder);
flowToDeploy.ResultProperties[ResultPropertyName_RuntimeConfigFolder] = flowToDeploy.GetTokenString(TokenName_RuntimeConfigFolder);
await Task.Yield();
return("done");
}
/// <summary>
/// Delete the runtime job configs
/// </summary>
/// <param name="flowToDelete"></param>
/// <returns></returns>
public override async Task <string> Delete(FlowDeploymentSession flowToDelete)
{
var flowConfig = flowToDelete.Config;
var runtimeConfigsFolder = flowConfig.GetJobConfigDestinationFolder();
flowToDelete.SetStringToken(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder, runtimeConfigsFolder);
var folderToDelete = flowToDelete.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
return(await this.JobData.DeleteConfigs(folderToDelete));
}
/// <summary>
/// Generate and set the info for the input schema file which will be used to generate JobConfig
/// </summary>
/// <returns></returns>
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
if (runtimeKeyVaultName != "local")
{
var secretName = $"{config.Name}-inputschemafile";
Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType);
var schemaFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret);
}
else
{
flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, flowToDeploy.ResultProperties[PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder].ToString() + "/inputschema.json");
}
await Task.CompletedTask;
return("done");
}
private void PortConfigurationSetting(FlowDeploymentSession session, string settingName, bool isRequired = false)
{
if (Configuration.TryGet(settingName, out var value))
{
session.SetStringToken(settingName, value);
}
else
{
if (isRequired)
{
throw new ConfigGenerationException($"Required setting '{settingName}' is not found in global configuration");
}
else
{
session.SetNullToken(settingName);
}
}
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var guiConfig = flowToDeploy.Config?.GetGuiConfig();
if (guiConfig == null)
{
// If guiConfig is empty, get the number of executors from job common token and convert it to integer
var executorsString = flowToDeploy.Config?.CommonProcessor?.JobCommonTokens?.GetOrDefault("sparkJobNumExecutors", null);
if (executorsString != null)
{
if (!int.TryParse(executorsString, out int executorsInt))
{
throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{executorsString}'.");
}
flowToDeploy.SetObjectToken("sparkJobNumExecutorsInt", executorsInt);
}
return("no gui config, skipped");
}
// Setting TokenName_SparkJobNumExecutors
var numExecutorsString = guiConfig?.Process?.JobConfig?.JobNumExecutors;
if (!int.TryParse(numExecutorsString, out int numExecutors))
{
throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{numExecutorsString}'.");
}
flowToDeploy.SetObjectToken(TokenName_SparkJobNumExecutors, numExecutors);
// Setting TokenName_SparkJobJobExecutorMemory
var jobExecutorMemoryString = guiConfig?.Process?.JobConfig?.JobExecutorMemory;
if (!int.TryParse(jobExecutorMemoryString, out int jobExecutorMemory))
{
throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobExecutorMemory:'{jobExecutorMemoryString}'.");
}
flowToDeploy.SetStringToken(TokenName_SparkJobJobExecutorMemory, $"{jobExecutorMemory}m");
await Task.Yield();
return("done");
}
/// <summary>
/// Generate and set the info for the input schema file which will be used to generate JobConfig
/// </summary>
/// <returns></returns>
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var secretName = $"{config.Name}-inputschemafile";
Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType);
var schemaFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret);
await Task.CompletedTask;
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
string queries = string.Join("\n", guiConfig.Process?.Queries);
string ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name);
RulesCode rulesCode = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name);
Ensure.NotNull(rulesCode, "rulesCode");
// Save the rulesCode object for downstream processing
flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode);
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt");
var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code);
var secretName = $"{config.Name}-transform";
var transformFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, transformFilePath);
flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);
return("done");
}
// Do input eventhub/iothub settings related processing
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
var config = flowConfig?.GetGuiConfig()?.Input;
var inputType = config?.InputType?.ToLowerInvariant();
if (inputType == null)
{
return("eventhub/iothub input not defined, skipped");
}
if (inputType != Constants.InputType_EventHub && inputType != Constants.InputType_IoTHub && inputType != Constants.InputType_KafkaEventHub && inputType != Constants.InputType_Kafka && inputType != Constants.InputType_Blob)
{
return($"unsupported inputtype '{inputType}', skipped.");
}
var props = config.Properties;
Ensure.NotNull(props, "flowConfig.Gui.Input.Properties");
var connectionString = props.InputEventhubConnection;
flowToDeploy.SetStringToken(TokenName_InputEventHubConnectionString, connectionString);
var resolvedConnectionString = await KeyVaultClient.ResolveSecretUriAsync(connectionString);
var hubInfo = ConnectionStringParser.ParseEventHub(resolvedConnectionString);
if (inputType == Constants.InputType_Kafka || inputType == Constants.InputType_KafkaEventHub)
{
hubInfo.Name = this.NormalizeEventNames(config.Properties.InputEventHubName);
}
var consumerGroupName = flowConfig.Name;
// Create consumer group only if the resource creation flag is set. This is to support scenario where EventHub
// is in a different subscription than where services are deployed
if (Configuration[Constants.ConfigSettingName_ResourceCreation].ToLower(CultureInfo.InvariantCulture) == "true" && (inputType == Constants.InputType_EventHub || inputType == Constants.InputType_IoTHub))
{
var serviceKeyVaultName = Configuration[Constants.ConfigSettingName_ServiceKeyVaultName];
Ensure.NotNull(serviceKeyVaultName, "serviceKeyVaultName");
var resolvedSecretKey = await KeyVaultClient.GetSecretFromKeyVaultAsync(serviceKeyVaultName, Configuration[Constants.ConfigSettingName_SecretPrefix] + "clientsecret");
var clientId = Configuration[Constants.ConfigSettingName_ConfigGenClientId];
var tenantId = Configuration[Constants.ConfigSettingName_ConfigGenTenantId];
var inputSubscriptionId = string.IsNullOrEmpty(config?.Properties?.InputSubscriptionId) ? await KeyVaultClient.ResolveSecretUriAsync(flowToDeploy.GetTokenString(TokenName_InputEventHubSubscriptionId)) : await KeyVaultClient.ResolveSecretUriAsync(config?.Properties?.InputSubscriptionId);
var inputResourceGroupName = string.IsNullOrEmpty(config?.Properties?.InputResourceGroup) ? flowToDeploy.GetTokenString(TokenName_InputEventHubResourceGroupName) : await KeyVaultClient.ResolveSecretUriAsync(config?.Properties?.InputResourceGroup);
Result result = null;
switch (inputType)
{
case Constants.InputType_EventHub:
case Constants.InputType_KafkaEventHub:
//Check for required parameters
if (string.IsNullOrEmpty(hubInfo.Namespace) || string.IsNullOrEmpty(hubInfo.Name))
{
throw new ConfigGenerationException("Could not parse Event Hub connection string; please check input.");
}
result = await EventHubUtil.CreateEventHubConsumerGroups(
clientId : clientId,
tenantId : tenantId,
secretKey : resolvedSecretKey,
subscriptionId : inputSubscriptionId,
resourceGroupName : inputResourceGroupName,
hubNamespace : hubInfo.Namespace,
hubNames : hubInfo.Name,
consumerGroupName : consumerGroupName);
break;
case Constants.InputType_IoTHub:
//Check for required parameters
if (string.IsNullOrEmpty(hubInfo.Name))
{
throw new ConfigGenerationException("Could not parse IoT Hub connection string; please check input.");
}
result = await EventHubUtil.CreateIotHubConsumerGroup(
clientId : clientId,
tenantId : tenantId,
secretKey : resolvedSecretKey,
subscriptionId : inputSubscriptionId,
resourceGroupName : inputResourceGroupName,
hubName : hubInfo.Name,
consumerGroupName : consumerGroupName);
break;
default:
throw new ConfigGenerationException($"unexpected inputtype '{inputType}'.");
}
Ensure.IsSuccessResult(result);
}
flowToDeploy.SetStringToken(TokenName_InputEventHubConsumerGroup, consumerGroupName);
flowToDeploy.SetStringToken(TokenName_InputEventHubs, hubInfo.Name);
var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;
var checkpointDir = (sparkType == Constants.SparkTypeDataBricks) ? Configuration.GetOrDefault(ConfigSettingName_InputEventHubCheckpointDir, "dbfs:/mycluster/datax/direct/${name}/") : Configuration.GetOrDefault(ConfigSettingName_InputEventHubCheckpointDir, "hdfs://mycluster/datax/direct/${name}/");
flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointDir, checkpointDir);
var intervalInSeconds = props?.WindowDuration;
flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointInterval, intervalInSeconds);
flowToDeploy.SetObjectToken(TokenName_InputEventHubMaxRate, props.MaxRate);
var flushCheckpointsString = Configuration.GetOrDefault(ConfigSettingName_InputEventHubFlushExistingCheckpoints, "False");
bool.TryParse(flushCheckpointsString, out bool flushExistingCheckpoints);
flowToDeploy.SetObjectToken(TokenName_InputEventHubFlushExistingCheckpoints, flushExistingCheckpoints);
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var guiConfig = flowToDeploy.Config?.GetGuiConfig();
if (guiConfig == null)
{
// If guiConfig is empty, get the number of executors from job common token and convert it to integer
var executorsString = flowToDeploy.Config?.CommonProcessor?.JobCommonTokens?.GetOrDefault("sparkJobNumExecutors", null);
if (executorsString != null)
{
if (!int.TryParse(executorsString, out int executorsInt))
{
throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{executorsString}'.");
}
flowToDeploy.SetObjectToken("sparkJobNumExecutorsInt", executorsInt);
}
return("no gui config, skipped");
}
// Setting TokenName_SparkJobNumExecutors
var numExecutorsString = guiConfig?.Process?.JobConfig?.JobNumExecutors;
if (!int.TryParse(numExecutorsString, out int numExecutors))
{
throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{numExecutorsString}'.");
}
flowToDeploy.SetObjectToken(TokenName_SparkJobNumExecutors, numExecutors);
// Setting TokenName_SparkJobJobExecutorMemory
var jobExecutorMemoryString = guiConfig?.Process?.JobConfig?.JobExecutorMemory;
if (!int.TryParse(jobExecutorMemoryString, out int jobExecutorMemory))
{
throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobExecutorMemory:'{jobExecutorMemoryString}'.");
}
flowToDeploy.SetStringToken(TokenName_SparkJobJobExecutorMemory, $"{jobExecutorMemory}m");
var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;
if (sparkType == Config.ConfigDataModel.Constants.SparkTypeDataBricks)
{
// Setting TokenName_SparkJobDatabricksMinWorkers
var jobDatabricksMinWorkersString = guiConfig?.Process?.JobConfig?.JobDatabricksMinWorkers;
if (!int.TryParse(jobDatabricksMinWorkersString, out int jobDatabricksMinWorkers))
{
throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobDatabricksMinWorkers:'{jobDatabricksMinWorkersString}'.");
}
flowToDeploy.SetStringToken(TokenName_SparkJobDatabricksMinWorkers, $"{jobDatabricksMinWorkers}");
// Setting TokenName_SparkJobDatabricksMaxWorkers
var jobDatabricksMaxWorkersString = guiConfig?.Process?.JobConfig?.JobDatabricksMaxWorkers;
if (!int.TryParse(jobDatabricksMaxWorkersString, out int jobDatabricksMaxWorkers))
{
throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobDatabricksMaxWorkers:'{jobDatabricksMaxWorkersString}'.");
}
flowToDeploy.SetStringToken(TokenName_SparkJobDatabricksMaxWorkers, $"{jobDatabricksMaxWorkers}");
// Setting TokenName_DatabricksToken
var jobDatabricksTokenString = guiConfig?.DatabricksToken;
flowToDeploy.SetStringToken(TokenName_DatabricksToken, $"{jobDatabricksTokenString}");
// Setting TokenName_SparkJobDatabricksAutoScale
var jobDatabricksAutoScaleString = guiConfig?.Process?.JobConfig?.JobDatabricksAutoScale;
flowToDeploy.SetStringToken(TokenName_SparkJobDatabricksAutoScale, $"{jobDatabricksAutoScaleString}");
}
await Task.Yield();
return("done");
}