/// <summary> /// Generate and set the info for the projection file which will be used to generate JobConfig /// </summary> /// <returns></returns> public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder); Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder"); var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName); Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName"); if (runtimeKeyVaultName != "local") { var secretName = $"{config.Name}-projectionfile"; Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType); var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType); var projectionFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix); flowToDeploy.SetObjectToken(TokenName_ProjectionFiles, new string[] { projectionFileSecret }); } else { flowToDeploy.SetObjectToken(TokenName_ProjectionFiles, new string[] { flowToDeploy.ResultProperties[PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder].ToString() + "/projection.txt" }); } await Task.CompletedTask; return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { //TODO: create eventhub consumer group var flowConfig = flowToDeploy.Config; var config = flowConfig?.GetGuiConfig()?.Input; //TODO: take care of other input types if (config != null && "iothub".Equals(config.InputType, StringComparison.InvariantCultureIgnoreCase)) { var props = config.Properties; if (props != null) { var connectionString = props.InputEventhubConnection; flowToDeploy.SetStringToken(TokenName_InputEventHubConnectionString, connectionString); // TODO: figure out the right value flowToDeploy.SetStringToken(TokenName_InputEventHubConsumerGroup, flowConfig.Name); flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointDir, $"hdfs://mycluster/dataxdirect/{JobMetadata.TokenPlaceHolder_JobName}/eventhub/checkpoints"); var intervalInSeconds = props?.WindowDuration; flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointInterval, intervalInSeconds); flowToDeploy.SetObjectToken(TokenName_InputEventHubMaxRate, props.MaxRate); flowToDeploy.SetObjectToken(TokenName_InputEventHubFlushExistingCheckpoints, true); } } await Task.Yield(); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject); Ensure.NotNull(rulesCode, "rulesCode"); Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot"); Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics"); var stateTables = rulesCode.AccumlationTables.Select(t => { return(new StateTableSpec() { Name = t.Key, Schema = t.Value, Location = ConstructStateTablePath(guiConfig.Name, t.Key) }); }).ToArray(); flowToDeploy.SetObjectToken(TokenName_StateTables, stateTables); await Task.Yield(); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } var rulesCode = flowToDeploy.GetAttachment <RulesCode>(GenerateTransformFile.AttachmentName_CodeGenObject); Ensure.NotNull(rulesCode, "rulesCode"); Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot"); Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics"); var timewindows = rulesCode.TimeWindows.Select(t => { return(new TimeWindowSpec() { Name = t.Key, WindowDuration = t.Value }); }).ToArray(); flowToDeploy.SetObjectToken(TokenName_TimeWindows, timewindows); await Task.Yield(); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped"); } var referenceData = guiConfig?.Input?.ReferenceData ?? Array.Empty <FlowGuiReferenceData>(); var specs = referenceData.Select(rd => { var props = rd.Properties; Ensure.NotNull(props, $"guiConfig.input.referenceData['{rd.Id}'].properties"); Ensure.NotNull(rd.Type, $"guiConfig.input.referenceData['{rd.Id}'].type"); Ensure.NotNull(props.Path, $"guiConfig.input.referenceData['{rd.Id}'].properties.path"); return(new ReferenceDataSpec() { Name = rd.Id, Path = props.Path, Format = rd.Type, Delimiter = props.Delimiter, Header = props.Header.ToString() }); }).ToArray(); flowToDeploy.SetObjectToken(TokenName_ReferenceData, specs); await Task.Yield(); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } var projectColumns = guiConfig.Input?.Properties?.NormalizationSnippet?.Trim('\t', ' ', '\r', '\n'); //TODO: make the hardcoded "Raw.*" configurable? var finalProjections = string.IsNullOrEmpty(projectColumns) ? "Raw.*" : projectColumns; var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder); Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder"); var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName); Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName"); var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "projection.txt"); var savedFile = await RuntimeStorage.SaveFile(filePath, finalProjections); var secretName = $"{config.Name}-projectionfile"; var savedSecretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, savedFile, Configuration[Constants.ConfigSettingName_SparkType]); flowToDeploy.SetObjectToken(TokenName_ProjectionFiles, new string[] { savedSecretId }); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var guiConfig = flowToDeploy.Config?.GetGuiConfig(); if (guiConfig == null) { // If guiConfig is empty, get the number of executors from job common token and convert it to integer var executorsString = flowToDeploy.Config?.CommonProcessor?.JobCommonTokens?.GetOrDefault("sparkJobNumExecutors", null); if (executorsString != null) { if (!int.TryParse(executorsString, out int executorsInt)) { throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{executorsString}'."); } flowToDeploy.SetObjectToken("sparkJobNumExecutorsInt", executorsInt); } return("no gui config, skipped"); } // Setting TokenName_SparkJobNumExecutors var numExecutorsString = guiConfig?.Process?.JobConfig?.JobNumExecutors; if (!int.TryParse(numExecutorsString, out int numExecutors)) { throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{numExecutorsString}'."); } flowToDeploy.SetObjectToken(TokenName_SparkJobNumExecutors, numExecutors); // Setting TokenName_SparkJobJobExecutorMemory var jobExecutorMemoryString = guiConfig?.Process?.JobConfig?.JobExecutorMemory; if (!int.TryParse(jobExecutorMemoryString, out int jobExecutorMemory)) { throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobExecutorMemory:'{jobExecutorMemoryString}'."); } flowToDeploy.SetStringToken(TokenName_SparkJobJobExecutorMemory, $"{jobExecutorMemory}m"); await Task.Yield(); return("done"); }
public override Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var functions = config.GetGuiConfig()?.Process?.Functions; if (functions == null) { return(Task.FromResult("no functions defined in gui input, skipped.")); } var jarUDFs = GetJarUdfDefinitions(functions, "jarUDF"); flowToDeploy.SetObjectToken(TokenName_JarUDFs, jarUDFs); var jarUDAFs = GetJarUdfDefinitions(functions, "jarUDAF"); flowToDeploy.SetObjectToken(TokenName_JarUDAFs, jarUDAFs); return(Task.FromResult("done")); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped"); } var rulesCode = flowToDeploy.GetAttachment <RulesCode>(GenerateTransformFile.AttachmentName_CodeGenObject); Ensure.NotNull(rulesCode, "rulesCode"); Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot"); Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics"); var outputs = await ProcessOutputs(guiConfig.Outputs, rulesCode, config.Name); flowToDeploy.SetObjectToken(TokenName_Outputs, outputs); return("done"); }
// Do input eventhub/iothub settings related processing public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var flowConfig = flowToDeploy.Config; var config = flowConfig?.GetGuiConfig()?.Input; var inputType = config?.InputType?.ToLowerInvariant(); if (inputType == null) { return("eventhub/iothub input not defined, skipped"); } if (inputType != Constants.InputType_EventHub && inputType != Constants.InputType_IoTHub && inputType != Constants.InputType_KafkaEventHub && inputType != Constants.InputType_Kafka && inputType != Constants.InputType_Blob) { return($"unsupported inputtype '{inputType}', skipped."); } var props = config.Properties; Ensure.NotNull(props, "flowConfig.Gui.Input.Properties"); var connectionString = props.InputEventhubConnection; flowToDeploy.SetStringToken(TokenName_InputEventHubConnectionString, connectionString); var resolvedConnectionString = await KeyVaultClient.ResolveSecretUriAsync(connectionString); var hubInfo = ConnectionStringParser.ParseEventHub(resolvedConnectionString); if (inputType == Constants.InputType_Kafka || inputType == Constants.InputType_KafkaEventHub) { hubInfo.Name = this.NormalizeEventNames(config.Properties.InputEventHubName); } var consumerGroupName = flowConfig.Name; // Create consumer group only if the resource creation flag is set. This is to support scenario where EventHub // is in a different subscription than where services are deployed if (Configuration[Constants.ConfigSettingName_ResourceCreation].ToLower(CultureInfo.InvariantCulture) == "true" && (inputType == Constants.InputType_EventHub || inputType == Constants.InputType_IoTHub)) { var serviceKeyVaultName = Configuration[Constants.ConfigSettingName_ServiceKeyVaultName]; Ensure.NotNull(serviceKeyVaultName, "serviceKeyVaultName"); var resolvedSecretKey = await KeyVaultClient.GetSecretFromKeyVaultAsync(serviceKeyVaultName, Configuration[Constants.ConfigSettingName_SecretPrefix] + "clientsecret"); var clientId = Configuration[Constants.ConfigSettingName_ConfigGenClientId]; var tenantId = Configuration[Constants.ConfigSettingName_ConfigGenTenantId]; var inputSubscriptionId = string.IsNullOrEmpty(config?.Properties?.InputSubscriptionId) ? await KeyVaultClient.ResolveSecretUriAsync(flowToDeploy.GetTokenString(TokenName_InputEventHubSubscriptionId)) : await KeyVaultClient.ResolveSecretUriAsync(config?.Properties?.InputSubscriptionId); var inputResourceGroupName = string.IsNullOrEmpty(config?.Properties?.InputResourceGroup) ? flowToDeploy.GetTokenString(TokenName_InputEventHubResourceGroupName) : await KeyVaultClient.ResolveSecretUriAsync(config?.Properties?.InputResourceGroup); Result result = null; switch (inputType) { case Constants.InputType_EventHub: case Constants.InputType_KafkaEventHub: //Check for required parameters if (string.IsNullOrEmpty(hubInfo.Namespace) || string.IsNullOrEmpty(hubInfo.Name)) { throw new ConfigGenerationException("Could not parse Event Hub connection string; please check input."); } result = await EventHubUtil.CreateEventHubConsumerGroups( clientId : clientId, tenantId : tenantId, secretKey : resolvedSecretKey, subscriptionId : inputSubscriptionId, resourceGroupName : inputResourceGroupName, hubNamespace : hubInfo.Namespace, hubNames : hubInfo.Name, consumerGroupName : consumerGroupName); break; case Constants.InputType_IoTHub: //Check for required parameters if (string.IsNullOrEmpty(hubInfo.Name)) { throw new ConfigGenerationException("Could not parse IoT Hub connection string; please check input."); } result = await EventHubUtil.CreateIotHubConsumerGroup( clientId : clientId, tenantId : tenantId, secretKey : resolvedSecretKey, subscriptionId : inputSubscriptionId, resourceGroupName : inputResourceGroupName, hubName : hubInfo.Name, consumerGroupName : consumerGroupName); break; default: throw new ConfigGenerationException($"unexpected inputtype '{inputType}'."); } Ensure.IsSuccessResult(result); } flowToDeploy.SetStringToken(TokenName_InputEventHubConsumerGroup, consumerGroupName); flowToDeploy.SetStringToken(TokenName_InputEventHubs, hubInfo.Name); var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null; var checkpointDir = (sparkType == Constants.SparkTypeDataBricks) ? Configuration.GetOrDefault(ConfigSettingName_InputEventHubCheckpointDir, "dbfs:/mycluster/datax/direct/${name}/") : Configuration.GetOrDefault(ConfigSettingName_InputEventHubCheckpointDir, "hdfs://mycluster/datax/direct/${name}/"); flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointDir, checkpointDir); var intervalInSeconds = props?.WindowDuration; flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointInterval, intervalInSeconds); flowToDeploy.SetObjectToken(TokenName_InputEventHubMaxRate, props.MaxRate); var flushCheckpointsString = Configuration.GetOrDefault(ConfigSettingName_InputEventHubFlushExistingCheckpoints, "False"); bool.TryParse(flushCheckpointsString, out bool flushExistingCheckpoints); flowToDeploy.SetObjectToken(TokenName_InputEventHubFlushExistingCheckpoints, flushExistingCheckpoints); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var guiConfig = flowToDeploy.Config?.GetGuiConfig(); if (guiConfig == null) { // If guiConfig is empty, get the number of executors from job common token and convert it to integer var executorsString = flowToDeploy.Config?.CommonProcessor?.JobCommonTokens?.GetOrDefault("sparkJobNumExecutors", null); if (executorsString != null) { if (!int.TryParse(executorsString, out int executorsInt)) { throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{executorsString}'."); } flowToDeploy.SetObjectToken("sparkJobNumExecutorsInt", executorsInt); } return("no gui config, skipped"); } // Setting TokenName_SparkJobNumExecutors var numExecutorsString = guiConfig?.Process?.JobConfig?.JobNumExecutors; if (!int.TryParse(numExecutorsString, out int numExecutors)) { throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{numExecutorsString}'."); } flowToDeploy.SetObjectToken(TokenName_SparkJobNumExecutors, numExecutors); // Setting TokenName_SparkJobJobExecutorMemory var jobExecutorMemoryString = guiConfig?.Process?.JobConfig?.JobExecutorMemory; if (!int.TryParse(jobExecutorMemoryString, out int jobExecutorMemory)) { throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobExecutorMemory:'{jobExecutorMemoryString}'."); } flowToDeploy.SetStringToken(TokenName_SparkJobJobExecutorMemory, $"{jobExecutorMemory}m"); var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null; if (sparkType == Config.ConfigDataModel.Constants.SparkTypeDataBricks) { // Setting TokenName_SparkJobDatabricksMinWorkers var jobDatabricksMinWorkersString = guiConfig?.Process?.JobConfig?.JobDatabricksMinWorkers; if (!int.TryParse(jobDatabricksMinWorkersString, out int jobDatabricksMinWorkers)) { throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobDatabricksMinWorkers:'{jobDatabricksMinWorkersString}'."); } flowToDeploy.SetStringToken(TokenName_SparkJobDatabricksMinWorkers, $"{jobDatabricksMinWorkers}"); // Setting TokenName_SparkJobDatabricksMaxWorkers var jobDatabricksMaxWorkersString = guiConfig?.Process?.JobConfig?.JobDatabricksMaxWorkers; if (!int.TryParse(jobDatabricksMaxWorkersString, out int jobDatabricksMaxWorkers)) { throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobDatabricksMaxWorkers:'{jobDatabricksMaxWorkersString}'."); } flowToDeploy.SetStringToken(TokenName_SparkJobDatabricksMaxWorkers, $"{jobDatabricksMaxWorkers}"); // Setting TokenName_DatabricksToken var jobDatabricksTokenString = guiConfig?.DatabricksToken; flowToDeploy.SetStringToken(TokenName_DatabricksToken, $"{jobDatabricksTokenString}"); // Setting TokenName_SparkJobDatabricksAutoScale var jobDatabricksAutoScaleString = guiConfig?.Process?.JobConfig?.JobDatabricksAutoScale; flowToDeploy.SetStringToken(TokenName_SparkJobDatabricksAutoScale, $"{jobDatabricksAutoScaleString}"); } await Task.Yield(); return("done"); }