Beispiel #1
0
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            //TODO: create eventhub consumer group
            var flowConfig = flowToDeploy.Config;
            var config     = flowConfig?.GetGuiConfig()?.Input;

            //TODO: take care of other input types
            if (config != null && "iothub".Equals(config.InputType, StringComparison.InvariantCultureIgnoreCase))
            {
                var props = config.Properties;
                if (props != null)
                {
                    var connectionString = props.InputEventhubConnection;
                    flowToDeploy.SetStringToken(TokenName_InputEventHubConnectionString, connectionString);

                    // TODO: figure out the right value
                    flowToDeploy.SetStringToken(TokenName_InputEventHubConsumerGroup, flowConfig.Name);
                    flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointDir, $"hdfs://mycluster/dataxdirect/{JobMetadata.TokenPlaceHolder_JobName}/eventhub/checkpoints");

                    var intervalInSeconds = props?.WindowDuration;
                    flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointInterval, intervalInSeconds);
                    flowToDeploy.SetObjectToken(TokenName_InputEventHubMaxRate, props.MaxRate);
                    flowToDeploy.SetObjectToken(TokenName_InputEventHubFlushExistingCheckpoints, true);
                }
            }
            await Task.Yield();

            return("done");
        }
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var config    = flowToDeploy.Config;
            var guiConfig = config.GetGuiConfig();

            flowToDeploy.SetStringToken(TokenName_InputStreamingCheckpointDir, $"hdfs://mycluster/dataxdirect/{JobMetadata.TokenPlaceHolder_JobName}/streaming/checkpoints");

            var intervalInSeconds = guiConfig?.Input?.Properties?.WindowDuration;

            flowToDeploy.SetStringToken(TokenName_InputStreamingInterval, intervalInSeconds);

            await Task.Yield();

            return("done");
        }
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var config    = flowToDeploy.Config;
            var guiConfig = config?.GetGuiConfig();

            if (guiConfig == null)
            {
                return("no gui input, skipped.");
            }

            var schema = guiConfig.Input?.Properties?.InputSchemaFile;

            Ensure.NotNull(schema, "guiConfig.input.properties.inputschemafile");

            var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);

            Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");

            var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);

            Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");

            var filePath   = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "inputschema.json");
            var schemaFile = await RuntimeStorage.SaveFile(filePath, schema);

            var secretName       = $"{config.Name}-inputschemafile";
            var schemaFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, schemaFile, Configuration[Constants.ConfigSettingName_SparkType]);

            flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret);

            return("done");
        }
        /// <summary>
        /// Generate and set the info for the transform file which will be used to generate JobConfig
        /// </summary>
        /// <returns></returns>
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var config    = flowToDeploy.Config;
            var guiConfig = config?.GetGuiConfig();

            if (guiConfig == null)
            {
                return("no gui input, skipped.");
            }
            string queries = string.Join("\n", guiConfig.Process?.Queries);

            string    ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name);
            RulesCode rulesCode       = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name);

            Ensure.NotNull(rulesCode, "rulesCode");

            // Save the rulesCode object for downstream processing
            flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode);

            var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);

            Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");

            var secretName = $"{config.Name}-transform";

            Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
            var uriPrefix           = KeyVaultClient.GetUriPrefix(sparkType);
            var transformFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);

            flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);

            await Task.CompletedTask;

            return("done");
        }
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var config    = flowToDeploy.Config;
            var guiConfig = config.GetGuiConfig();
            var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;
            var inputStreamingCheckpointDirPrefix = (sparkType == Constants.SparkTypeDataBricks) ? Constants.PrefixDbfs : Constants.PrefixHdfs;

            flowToDeploy.SetStringToken(TokenName_InputStreamingCheckpointDir, $"{inputStreamingCheckpointDirPrefix}mycluster/dataxdirect/{JobMetadata.TokenPlaceHolder_JobName}/streaming/checkpoints");

            var intervalInSeconds = guiConfig?.Input?.Properties?.WindowDuration;

            flowToDeploy.SetStringToken(TokenName_InputStreamingInterval, intervalInSeconds);

            await Task.Yield();

            return("done");
        }
Beispiel #6
0
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var config    = flowToDeploy.Config;
            var guiConfig = config.GetGuiConfig();

            if (guiConfig == null)
            {
                return("no gui input, skipped.");
            }

            flowToDeploy.SetStringToken(TokenName_ProcessTimestampColumn, guiConfig.Process.TimestampColumn);
            flowToDeploy.SetStringToken(TokenName_ProcessWatermark, guiConfig.Process.Watermark);

            await Task.Yield();

            return("done");
        }
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var flowConfig = flowToDeploy.Config;

            // determine the runtime config folder
            var version = VersionGeneration.Next();

            flowToDeploy.SetStringToken(TokenName_ConfigVersion, version);
            var runtimeConfigFolder = this.JobData.FigureOutDestinationFolder(flowConfig.GetJobConfigDestinationFolder(), version);

            flowToDeploy.SetStringToken(TokenName_RuntimeConfigFolder, runtimeConfigFolder);
            flowToDeploy.ResultProperties[ResultPropertyName_RuntimeConfigFolder] = flowToDeploy.GetTokenString(TokenName_RuntimeConfigFolder);

            await Task.Yield();

            return("done");
        }
        /// <summary>
        /// Delete the runtime job configs
        /// </summary>
        /// <param name="flowToDelete"></param>
        /// <returns></returns>
        public override async Task <string> Delete(FlowDeploymentSession flowToDelete)
        {
            var flowConfig           = flowToDelete.Config;
            var runtimeConfigsFolder = flowConfig.GetJobConfigDestinationFolder();

            flowToDelete.SetStringToken(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder, runtimeConfigsFolder);
            var folderToDelete = flowToDelete.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);

            return(await this.JobData.DeleteConfigs(folderToDelete));
        }
        /// <summary>
        /// Generate and set the info for the input schema file which will be used to generate JobConfig
        /// </summary>
        /// <returns></returns>
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var config = flowToDeploy.Config;
            var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);

            Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
            if (runtimeKeyVaultName != "local")
            {
                var secretName = $"{config.Name}-inputschemafile";
                Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
                var uriPrefix        = KeyVaultClient.GetUriPrefix(sparkType);
                var schemaFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
                flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret);
            }
            else
            {
                flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, flowToDeploy.ResultProperties[PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder].ToString() + "/inputschema.json");
            }

            await Task.CompletedTask;

            return("done");
        }
 private void PortConfigurationSetting(FlowDeploymentSession session, string settingName, bool isRequired = false)
 {
     if (Configuration.TryGet(settingName, out var value))
     {
         session.SetStringToken(settingName, value);
     }
     else
     {
         if (isRequired)
         {
             throw new ConfigGenerationException($"Required setting '{settingName}' is not found in global configuration");
         }
         else
         {
             session.SetNullToken(settingName);
         }
     }
 }
Beispiel #11
0
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var guiConfig = flowToDeploy.Config?.GetGuiConfig();

            if (guiConfig == null)
            {
                // If guiConfig is empty, get the number of executors from job common token and convert it to integer
                var executorsString = flowToDeploy.Config?.CommonProcessor?.JobCommonTokens?.GetOrDefault("sparkJobNumExecutors", null);
                if (executorsString != null)
                {
                    if (!int.TryParse(executorsString, out int executorsInt))
                    {
                        throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{executorsString}'.");
                    }
                    flowToDeploy.SetObjectToken("sparkJobNumExecutorsInt", executorsInt);
                }
                return("no gui config, skipped");
            }

            // Setting TokenName_SparkJobNumExecutors
            var numExecutorsString = guiConfig?.Process?.JobConfig?.JobNumExecutors;

            if (!int.TryParse(numExecutorsString, out int numExecutors))
            {
                throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{numExecutorsString}'.");
            }

            flowToDeploy.SetObjectToken(TokenName_SparkJobNumExecutors, numExecutors);

            // Setting TokenName_SparkJobJobExecutorMemory
            var jobExecutorMemoryString = guiConfig?.Process?.JobConfig?.JobExecutorMemory;

            if (!int.TryParse(jobExecutorMemoryString, out int jobExecutorMemory))
            {
                throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobExecutorMemory:'{jobExecutorMemoryString}'.");
            }

            flowToDeploy.SetStringToken(TokenName_SparkJobJobExecutorMemory, $"{jobExecutorMemory}m");

            await Task.Yield();

            return("done");
        }
Beispiel #12
0
        /// <summary>
        /// Generate and set the info for the input schema file which will be used to generate JobConfig
        /// </summary>
        /// <returns></returns>
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var config = flowToDeploy.Config;
            var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);

            Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");

            var secretName = $"{config.Name}-inputschemafile";

            Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
            var uriPrefix        = KeyVaultClient.GetUriPrefix(sparkType);
            var schemaFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);

            flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret);

            await Task.CompletedTask;

            return("done");
        }
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var config    = flowToDeploy.Config;
            var guiConfig = config?.GetGuiConfig();

            if (guiConfig == null)
            {
                return("no gui input, skipped.");
            }
            string queries = string.Join("\n", guiConfig.Process?.Queries);

            string    ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name);
            RulesCode rulesCode       = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name);

            Ensure.NotNull(rulesCode, "rulesCode");

            // Save the rulesCode object for downstream processing
            flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode);

            var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);

            Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");

            var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);

            Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");

            var filePath          = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt");
            var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code);

            var secretName          = $"{config.Name}-transform";
            var transformFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, transformFilePath);

            flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);

            return("done");
        }
        // Do input eventhub/iothub settings related processing
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var flowConfig = flowToDeploy.Config;
            var config     = flowConfig?.GetGuiConfig()?.Input;

            var inputType = config?.InputType?.ToLowerInvariant();

            if (inputType == null)
            {
                return("eventhub/iothub input not defined, skipped");
            }

            if (inputType != Constants.InputType_EventHub && inputType != Constants.InputType_IoTHub && inputType != Constants.InputType_KafkaEventHub && inputType != Constants.InputType_Kafka && inputType != Constants.InputType_Blob)
            {
                return($"unsupported inputtype '{inputType}', skipped.");
            }

            var props = config.Properties;

            Ensure.NotNull(props, "flowConfig.Gui.Input.Properties");

            var connectionString = props.InputEventhubConnection;

            flowToDeploy.SetStringToken(TokenName_InputEventHubConnectionString, connectionString);

            var resolvedConnectionString = await KeyVaultClient.ResolveSecretUriAsync(connectionString);

            var hubInfo = ConnectionStringParser.ParseEventHub(resolvedConnectionString);

            if (inputType == Constants.InputType_Kafka || inputType == Constants.InputType_KafkaEventHub)
            {
                hubInfo.Name = this.NormalizeEventNames(config.Properties.InputEventHubName);
            }

            var consumerGroupName = flowConfig.Name;

            // Create consumer group only if the resource creation flag is set. This is to support scenario where EventHub
            // is in a different subscription than where services are deployed
            if (Configuration[Constants.ConfigSettingName_ResourceCreation].ToLower(CultureInfo.InvariantCulture) == "true" && (inputType == Constants.InputType_EventHub || inputType == Constants.InputType_IoTHub))
            {
                var serviceKeyVaultName = Configuration[Constants.ConfigSettingName_ServiceKeyVaultName];
                Ensure.NotNull(serviceKeyVaultName, "serviceKeyVaultName");
                var resolvedSecretKey = await KeyVaultClient.GetSecretFromKeyVaultAsync(serviceKeyVaultName, Configuration[Constants.ConfigSettingName_SecretPrefix] + "clientsecret");

                var clientId = Configuration[Constants.ConfigSettingName_ConfigGenClientId];
                var tenantId = Configuration[Constants.ConfigSettingName_ConfigGenTenantId];

                var inputSubscriptionId = string.IsNullOrEmpty(config?.Properties?.InputSubscriptionId) ? await KeyVaultClient.ResolveSecretUriAsync(flowToDeploy.GetTokenString(TokenName_InputEventHubSubscriptionId)) : await KeyVaultClient.ResolveSecretUriAsync(config?.Properties?.InputSubscriptionId);

                var inputResourceGroupName = string.IsNullOrEmpty(config?.Properties?.InputResourceGroup) ? flowToDeploy.GetTokenString(TokenName_InputEventHubResourceGroupName) : await KeyVaultClient.ResolveSecretUriAsync(config?.Properties?.InputResourceGroup);

                Result result = null;
                switch (inputType)
                {
                case Constants.InputType_EventHub:
                case Constants.InputType_KafkaEventHub:
                    //Check for required parameters
                    if (string.IsNullOrEmpty(hubInfo.Namespace) || string.IsNullOrEmpty(hubInfo.Name))
                    {
                        throw new ConfigGenerationException("Could not parse Event Hub connection string; please check input.");
                    }
                    result = await EventHubUtil.CreateEventHubConsumerGroups(
                        clientId : clientId,
                        tenantId : tenantId,
                        secretKey : resolvedSecretKey,
                        subscriptionId : inputSubscriptionId,
                        resourceGroupName : inputResourceGroupName,
                        hubNamespace : hubInfo.Namespace,
                        hubNames : hubInfo.Name,
                        consumerGroupName : consumerGroupName);

                    break;

                case Constants.InputType_IoTHub:
                    //Check for required parameters
                    if (string.IsNullOrEmpty(hubInfo.Name))
                    {
                        throw new ConfigGenerationException("Could not parse IoT Hub connection string; please check input.");
                    }
                    result = await EventHubUtil.CreateIotHubConsumerGroup(
                        clientId : clientId,
                        tenantId : tenantId,
                        secretKey : resolvedSecretKey,
                        subscriptionId : inputSubscriptionId,
                        resourceGroupName : inputResourceGroupName,
                        hubName : hubInfo.Name,
                        consumerGroupName : consumerGroupName);

                    break;

                default:
                    throw new ConfigGenerationException($"unexpected inputtype '{inputType}'.");
                }

                Ensure.IsSuccessResult(result);
            }

            flowToDeploy.SetStringToken(TokenName_InputEventHubConsumerGroup, consumerGroupName);
            flowToDeploy.SetStringToken(TokenName_InputEventHubs, hubInfo.Name);
            var sparkType     = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;
            var checkpointDir = (sparkType == Constants.SparkTypeDataBricks) ? Configuration.GetOrDefault(ConfigSettingName_InputEventHubCheckpointDir, "dbfs:/mycluster/datax/direct/${name}/") : Configuration.GetOrDefault(ConfigSettingName_InputEventHubCheckpointDir, "hdfs://mycluster/datax/direct/${name}/");

            flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointDir, checkpointDir);

            var intervalInSeconds = props?.WindowDuration;

            flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointInterval, intervalInSeconds);
            flowToDeploy.SetObjectToken(TokenName_InputEventHubMaxRate, props.MaxRate);

            var flushCheckpointsString = Configuration.GetOrDefault(ConfigSettingName_InputEventHubFlushExistingCheckpoints, "False");

            bool.TryParse(flushCheckpointsString, out bool flushExistingCheckpoints);
            flowToDeploy.SetObjectToken(TokenName_InputEventHubFlushExistingCheckpoints, flushExistingCheckpoints);

            return("done");
        }
        public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
        {
            var guiConfig = flowToDeploy.Config?.GetGuiConfig();

            if (guiConfig == null)

            {
                // If guiConfig is empty, get the number of executors from job common token and convert it to integer
                var executorsString = flowToDeploy.Config?.CommonProcessor?.JobCommonTokens?.GetOrDefault("sparkJobNumExecutors", null);
                if (executorsString != null)
                {
                    if (!int.TryParse(executorsString, out int executorsInt))
                    {
                        throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{executorsString}'.");
                    }
                    flowToDeploy.SetObjectToken("sparkJobNumExecutorsInt", executorsInt);
                }
                return("no gui config, skipped");
            }

            // Setting TokenName_SparkJobNumExecutors
            var numExecutorsString = guiConfig?.Process?.JobConfig?.JobNumExecutors;

            if (!int.TryParse(numExecutorsString, out int numExecutors))
            {
                throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{numExecutorsString}'.");
            }

            flowToDeploy.SetObjectToken(TokenName_SparkJobNumExecutors, numExecutors);

            // Setting TokenName_SparkJobJobExecutorMemory
            var jobExecutorMemoryString = guiConfig?.Process?.JobConfig?.JobExecutorMemory;

            if (!int.TryParse(jobExecutorMemoryString, out int jobExecutorMemory))
            {
                throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobExecutorMemory:'{jobExecutorMemoryString}'.");
            }

            flowToDeploy.SetStringToken(TokenName_SparkJobJobExecutorMemory, $"{jobExecutorMemory}m");
            var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;

            if (sparkType == Config.ConfigDataModel.Constants.SparkTypeDataBricks)
            {
                // Setting TokenName_SparkJobDatabricksMinWorkers
                var jobDatabricksMinWorkersString = guiConfig?.Process?.JobConfig?.JobDatabricksMinWorkers;
                if (!int.TryParse(jobDatabricksMinWorkersString, out int jobDatabricksMinWorkers))
                {
                    throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobDatabricksMinWorkers:'{jobDatabricksMinWorkersString}'.");
                }

                flowToDeploy.SetStringToken(TokenName_SparkJobDatabricksMinWorkers, $"{jobDatabricksMinWorkers}");

                // Setting TokenName_SparkJobDatabricksMaxWorkers
                var jobDatabricksMaxWorkersString = guiConfig?.Process?.JobConfig?.JobDatabricksMaxWorkers;
                if (!int.TryParse(jobDatabricksMaxWorkersString, out int jobDatabricksMaxWorkers))
                {
                    throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobDatabricksMaxWorkers:'{jobDatabricksMaxWorkersString}'.");
                }

                flowToDeploy.SetStringToken(TokenName_SparkJobDatabricksMaxWorkers, $"{jobDatabricksMaxWorkers}");

                // Setting TokenName_DatabricksToken
                var jobDatabricksTokenString = guiConfig?.DatabricksToken;
                flowToDeploy.SetStringToken(TokenName_DatabricksToken, $"{jobDatabricksTokenString}");

                // Setting TokenName_SparkJobDatabricksAutoScale
                var jobDatabricksAutoScaleString = guiConfig?.Process?.JobConfig?.JobDatabricksAutoScale;
                flowToDeploy.SetStringToken(TokenName_SparkJobDatabricksAutoScale, $"{jobDatabricksAutoScaleString}");
            }
            await Task.Yield();

            return("done");
        }