public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } var schema = guiConfig.Input?.Properties?.InputSchemaFile; Ensure.NotNull(schema, "guiConfig.input.properties.inputschemafile"); var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder); Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder"); var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName); Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName"); var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "inputschema.json"); var schemaFile = await RuntimeStorage.SaveFile(filePath, schema); var secretName = $"{config.Name}-inputschemafile"; var schemaFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, schemaFile, Configuration[Constants.ConfigSettingName_SparkType]); flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } var projectColumns = guiConfig.Input?.Properties?.NormalizationSnippet?.Trim('\t', ' ', '\r', '\n'); //TODO: make the hardcoded "Raw.*" configurable? var finalProjections = string.IsNullOrEmpty(projectColumns) ? "Raw.*" : projectColumns; var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder); Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder"); var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName); Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName"); var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "projection.txt"); var savedFile = await RuntimeStorage.SaveFile(filePath, finalProjections); var secretName = $"{config.Name}-projectionfile"; var savedSecretId = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, savedFile, Configuration[Constants.ConfigSettingName_SparkType]); flowToDeploy.SetObjectToken(TokenName_ProjectionFiles, new string[] { savedSecretId }); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { // Deploy job configs var jobs = flowToDeploy.GetJobs(); var deploymentTasks = jobs?.Select(async job => { var content = job.GetTokenString(GenerateJobConfig.TokenName_JobConfigContent); var filePath = job.GetTokenString(GenerateJobConfig.TokenName_JobConfigFilePath); if (content != null && filePath != null) { job.SparkJobConfigFilePath = await this.JobData.SaveFile(filePath, content); } else { job.SparkJobConfigFilePath = null; } return(job.SparkJobConfigFilePath); }); // Ensure all jobs configs are written successfully if (deploymentTasks == null) { return("no jobs, skipped"); } await Task.WhenAll(deploymentTasks); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped"); } var referenceData = guiConfig?.Input?.ReferenceData ?? Array.Empty <FlowGuiReferenceData>(); var specs = referenceData.Select(rd => { var props = rd.Properties; Ensure.NotNull(props, $"guiConfig.input.referenceData['{rd.Id}'].properties"); Ensure.NotNull(rd.Type, $"guiConfig.input.referenceData['{rd.Id}'].type"); Ensure.NotNull(props.Path, $"guiConfig.input.referenceData['{rd.Id}'].properties.path"); return(new ReferenceDataSpec() { Name = rd.Id, Path = props.Path, Format = rd.Type, Delimiter = props.Delimiter, Header = props.Header.ToString() }); }).ToArray(); flowToDeploy.SetObjectToken(TokenName_ReferenceData, specs); await Task.Yield(); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { // get a flattener var flattener = await this.ConfigFlatteners.GetDefault(); if (flattener == null) { return("no flattern config, skipped"); } // flatten each job config var jobs = flowToDeploy.GetJobs(); if (jobs == null) { return("no jobs, skipped"); } foreach (var job in jobs) { foreach (var jc in job.JobConfigs) { var jsonContent = job.Tokens.Resolve(jc.Content); if (jsonContent != null) { var json = JsonConfig.From(jsonContent); jc.Content = flattener.Flatten(json); } } } return("done"); }
/// <summary> /// Generate and set the info for the projection file which will be used to generate JobConfig /// </summary> /// <returns></returns> public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder); Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder"); var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName); Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName"); if (runtimeKeyVaultName != "local") { var secretName = $"{config.Name}-projectionfile"; Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType); var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType); var projectionFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix); flowToDeploy.SetObjectToken(TokenName_ProjectionFiles, new string[] { projectionFileSecret }); } else { flowToDeploy.SetObjectToken(TokenName_ProjectionFiles, new string[] { flowToDeploy.ResultProperties[PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder].ToString() + "/projection.txt" }); } await Task.CompletedTask; return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject); Ensure.NotNull(rulesCode, "rulesCode"); Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot"); Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics"); var stateTables = rulesCode.AccumlationTables.Select(t => { return(new StateTableSpec() { Name = t.Key, Schema = t.Value, Location = ConstructStateTablePath(guiConfig.Name, t.Key) }); }).ToArray(); flowToDeploy.SetObjectToken(TokenName_StateTables, stateTables); await Task.Yield(); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } var rulesCode = flowToDeploy.GetAttachment <RulesCode>(GenerateTransformFile.AttachmentName_CodeGenObject); Ensure.NotNull(rulesCode, "rulesCode"); Ensure.NotNull(rulesCode.MetricsRoot, "rulesCode.MetricsRoot"); Ensure.NotNull(rulesCode.MetricsRoot.metrics, "rulesCode.MetricsRoot.metrics"); var timewindows = rulesCode.TimeWindows.Select(t => { return(new TimeWindowSpec() { Name = t.Key, WindowDuration = t.Value }); }).ToArray(); flowToDeploy.SetObjectToken(TokenName_TimeWindows, timewindows); await Task.Yield(); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var flowConfig = flowToDeploy.Config; if (flowConfig.GetGuiConfig()?.Input?.Mode != Constants.InputMode_Batching) { return("done"); } // set the default job config var defaultJobConfig = JsonConfig.From(flowConfig.CommonProcessor?.Template); Ensure.NotNull(defaultJobConfig, "defaultJobConfig"); flowToDeploy.SetAttachment(TokenName__DefaultJobConfig, defaultJobConfig); // Deploy job configs var jobsToDeploy = flowToDeploy?.GetJobs(); if (jobsToDeploy != null) { foreach (var job in jobsToDeploy) { await GenerateJobConfigContent(job, job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder), defaultJobConfig).ConfigureAwait(false); } return("done"); } else { await Task.Yield(); return("no jobs, skipped"); } }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { //TODO: create eventhub consumer group var flowConfig = flowToDeploy.Config; var config = flowConfig?.GetGuiConfig()?.Input; //TODO: take care of other input types if (config != null && "iothub".Equals(config.InputType, StringComparison.InvariantCultureIgnoreCase)) { var props = config.Properties; if (props != null) { var connectionString = props.InputEventhubConnection; flowToDeploy.SetStringToken(TokenName_InputEventHubConnectionString, connectionString); // TODO: figure out the right value flowToDeploy.SetStringToken(TokenName_InputEventHubConsumerGroup, flowConfig.Name); flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointDir, $"hdfs://mycluster/dataxdirect/{JobMetadata.TokenPlaceHolder_JobName}/eventhub/checkpoints"); var intervalInSeconds = props?.WindowDuration; flowToDeploy.SetStringToken(TokenName_InputEventHubCheckpointInterval, intervalInSeconds); flowToDeploy.SetObjectToken(TokenName_InputEventHubMaxRate, props.MaxRate); flowToDeploy.SetObjectToken(TokenName_InputEventHubFlushExistingCheckpoints, true); } } await Task.Yield(); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var flowConfig = flowToDeploy.Config; // set the default job config var defaultJobConfig = JsonConfig.From(flowConfig.CommonProcessor?.Template); Ensure.NotNull(defaultJobConfig, "defaultJobConfig"); flowToDeploy.SetAttachment(ParameterObjectName_DefaultJobConfig, defaultJobConfig); // Deploy job configs var jobsToDeploy = flowToDeploy?.GetJobs(); if (jobsToDeploy != null) { foreach (var job in jobsToDeploy) { GenerateJobConfigContent(job, job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder), defaultJobConfig); } return("done"); } else { await Task.Yield(); return("no jobs, skipped"); } }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { if (flowToDeploy.Config.GetGuiConfig().Input?.Mode == Constants.InputMode_Batching) { var inputConfig = flowToDeploy.Config.GetGuiConfig(); var inputBatching = inputConfig.Input.Batch ?? Array.Empty <FlowGuiInputBatchInput>(); var specsTasks = inputBatching.Select(async rd => { var connectionString = await KeyVaultClient.ResolveSecretUriAsync(rd.Properties.Connection).ConfigureAwait(false); var inputPath = await KeyVaultClient.ResolveSecretUriAsync(rd.Properties.Path).ConfigureAwait(false); return(new InputBatchingSpec() { Name = ConfigHelper.ParseBlobAccountName(connectionString), Path = rd.Properties.Path, Format = rd.Properties.FormatType, CompressionType = rd.Properties.CompressionType, ProcessStartTime = "", ProcessEndTime = "", PartitionIncrement = GetPartitionIncrement(inputPath).ToString(CultureInfo.InvariantCulture), }); }).ToArray(); var specs = await Task.WhenAll(specsTasks).ConfigureAwait(false); flowToDeploy.SetAttachment(TokenName_InputBatching, specs); } return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var jobs = flowToDeploy.GetJobs(); if (jobs == null || !jobs.Where(j => j.JobConfigs.Any()).Any()) { return("no jobs, skipped"); } var config = flowToDeploy.Config; var rulesCode = flowToDeploy.GetAttachment <RulesCode>(PrepareTransformFile.AttachmentName_CodeGenObject); Ensure.NotNull(rulesCode, "rulesCode"); var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder); Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder"); var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt"); var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code); var transformFileSecret = flowToDeploy.GetTokenString(PrepareTransformFile.TokenName_TransformFile); await KeyVaultClient.SaveSecretAsync(transformFileSecret, transformFilePath); return("done"); }
/// <summary> /// Generate and set the info for the transform file which will be used to generate JobConfig /// </summary> /// <returns></returns> public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } string queries = string.Join("\n", guiConfig.Process?.Queries); string ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name); RulesCode rulesCode = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name); Ensure.NotNull(rulesCode, "rulesCode"); // Save the rulesCode object for downstream processing flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode); var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName); Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName"); var secretName = $"{config.Name}-transform"; Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType); var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType); var transformFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix); flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret); await Task.CompletedTask; return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { await Task.Yield(); // Go through a pass of validation to ensure required fields are present var result = this.ValidateFlowConfig(flowToDeploy.Config); return($"result is {result.IsSuccess}"); }
/// <summary> /// Delete the runtime job configs /// </summary> /// <param name="flowToDelete"></param> /// <returns></returns> public override async Task <string> Delete(FlowDeploymentSession flowToDelete) { var flowConfig = flowToDelete.Config; var runtimeConfigsFolder = flowConfig.GetJobConfigDestinationFolder(); flowToDelete.SetStringToken(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder, runtimeConfigsFolder); var folderToDelete = flowToDelete.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder); return(await this.JobData.DeleteConfigs(folderToDelete)); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var guiConfig = flowToDeploy.Config.GetGuiConfig(); if (guiConfig?.Input?.Mode == Constants.InputMode_Batching) { var jobConfigs = flowToDeploy.GetJobs().First().JobConfigs; var reConfigs = jobConfigs.Where(j => !j.IsOneTime).ToList(); var otConfigs = jobConfigs.Where(j => j.IsOneTime).ToList(); Assert.AreEqual(expected: 2, actual: reConfigs.Count); Assert.AreEqual(expected: 3, actual: otConfigs.Count); var startTimeConfig = (DateTime)guiConfig.BatchList[0].Properties.StartTime; var curTime = startTimeConfig.AddDays(1); var normalizedStartTime = startTimeConfig.Add(-startTimeConfig.TimeOfDay); TimeSpan interval = new TimeSpan(1, 0, 0, 0); var expected = normalizedStartTime; foreach (var c in reConfigs) { var actualStart = DateTime.Parse(c.ProcessStartTime).ToUniversalTime(); var actualEnd = DateTime.Parse(c.ProcessEndTime).ToUniversalTime(); var window = (actualEnd - actualStart).TotalSeconds; Assert.AreEqual(expected: expected.AddDays(-2), actual: actualStart, message: $"StartTime:{actualStart}"); Assert.AreEqual(expected: expected.AddDays(1).AddSeconds(-1), actual: actualEnd, message: $"EndTime:{actualEnd}"); Assert.AreEqual(expected: 259199, actual: window, message: $"Window:{window}"); Assert.IsTrue(actualStart < curTime); expected = expected.Add(interval); } var lastScheduleTime = DateTime.Parse(reConfigs.Last().ProcessStartTime).ToUniversalTime(); Assert.IsTrue(lastScheduleTime < curTime); expected = normalizedStartTime; foreach (var c in otConfigs) { var actualStart = DateTime.Parse(c.ProcessStartTime).ToUniversalTime(); var actualEnd = DateTime.Parse(c.ProcessEndTime).ToUniversalTime(); var window = (actualEnd - actualStart).TotalSeconds; Assert.AreEqual(expected: expected, actual: actualStart, message: $"StartTime:{actualStart}"); Assert.AreEqual(expected: expected.AddDays(1).AddSeconds(-1), actual: actualEnd, message: $"EndTime:{actualEnd}"); Assert.AreEqual(expected: 86399, actual: window, message: $"Window:{window}"); expected = expected.Add(interval); } lastScheduleTime = DateTime.Parse(otConfigs.Last().ProcessStartTime).ToUniversalTime(); Assert.IsTrue(lastScheduleTime > curTime); } return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var expectedConfigContent = await File.ReadAllTextAsync(@"Resource\jobConfig.json"); var expectedJson = JsonConfig.From(expectedConfigContent); var actualContentContent = flowToDeploy.GetJobs().First().GetTokenString(GenerateJobConfig.TokenName_JobConfigContent); var actualJson = JsonConfig.From(actualContentContent); foreach (var match in JsonConfig.Match(expectedJson, actualJson)) { Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}"); } return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config.GetGuiConfig(); flowToDeploy.SetStringToken(TokenName_InputStreamingCheckpointDir, $"hdfs://mycluster/dataxdirect/{JobMetadata.TokenPlaceHolder_JobName}/streaming/checkpoints"); var intervalInSeconds = guiConfig?.Input?.Properties?.WindowDuration; flowToDeploy.SetStringToken(TokenName_InputStreamingInterval, intervalInSeconds); await Task.Yield(); return("done"); }
public override Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var functions = config.GetGuiConfig()?.Process?.Functions; if (functions == null) { return(Task.FromResult("no functions defined in gui input, skipped.")); } var azureFuncs = GetAzureFuncDefinitions(functions); flowToDeploy.SetObjectToken(TokenName_AzureFunctions, azureFuncs); return(Task.FromResult("done")); }
/// <summary> /// Delete the job config /// </summary> /// <param name="flowToDelete"></param> /// <returns></returns> public override async Task <string> Delete(FlowDeploymentSession flowToDelete) { var jobNames = flowToDelete?.Config?.JobNames; if (jobNames == null) { return("skipped"); } foreach (var jobName in jobNames) { await SparkJobData.DeleteByName(jobName); } return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } flowToDeploy.SetStringToken(TokenName_ProcessTimestampColumn, guiConfig.Process.TimestampColumn); flowToDeploy.SetStringToken(TokenName_ProcessWatermark, guiConfig.Process.Watermark); await Task.Yield(); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var guiConfig = config.GetGuiConfig(); var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null; var inputStreamingCheckpointDirPrefix = (sparkType == Constants.SparkTypeDataBricks) ? Constants.PrefixDbfs : Constants.PrefixHdfs; flowToDeploy.SetStringToken(TokenName_InputStreamingCheckpointDir, $"{inputStreamingCheckpointDirPrefix}mycluster/dataxdirect/{JobMetadata.TokenPlaceHolder_JobName}/streaming/checkpoints"); var intervalInSeconds = guiConfig?.Input?.Properties?.WindowDuration; flowToDeploy.SetStringToken(TokenName_InputStreamingInterval, intervalInSeconds); await Task.Yield(); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var flowConfig = flowToDeploy.Config; // determine the runtime config folder var version = VersionGeneration.Next(); flowToDeploy.SetStringToken(TokenName_ConfigVersion, version); var runtimeConfigFolder = this.JobData.FigureOutDestinationFolder(flowConfig.GetJobConfigDestinationFolder(), version); flowToDeploy.SetStringToken(TokenName_RuntimeConfigFolder, runtimeConfigFolder); flowToDeploy.ResultProperties[ResultPropertyName_RuntimeConfigFolder] = flowToDeploy.GetTokenString(TokenName_RuntimeConfigFolder); await Task.Yield(); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var flowConfig = flowToDeploy.Config; var defaultConfig = await FlowData.GetFlowConfigByInputType(flowConfig.GetGuiConfig().Input.InputType, flowToDeploy.Tokens); if (defaultConfig == null) { return("defaultConfig is null, skipped"); } var newConfig = flowConfig.RebaseOn(defaultConfig); flowToDeploy.UpdateFlowConfig(newConfig); flowToDeploy.SetAttachment(AttachementName_DefaultFlowConfig, defaultConfig); return("done"); }
private void PortConfigurationSetting(FlowDeploymentSession session, string settingName, bool isRequired = false) { if (Configuration.TryGet(settingName, out var value)) { session.SetStringToken(settingName, value); } else { if (isRequired) { throw new ConfigGenerationException($"Required setting '{settingName}' is not found in global configuration"); } else { session.SetNullToken(settingName); } } }
/// <summary> /// Generate and set the info for the input schema file which will be used to generate JobConfig /// </summary> /// <returns></returns> public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var config = flowToDeploy.Config; var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName); Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName"); var secretName = $"{config.Name}-inputschemafile"; Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType); var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType); var schemaFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix); flowToDeploy.SetStringToken(TokenName_InputSchemaFilePath, schemaFileSecret); await Task.CompletedTask; return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var guiConfig = flowToDeploy.Config?.GetGuiConfig(); if (guiConfig == null) { // If guiConfig is empty, get the number of executors from job common token and convert it to integer var executorsString = flowToDeploy.Config?.CommonProcessor?.JobCommonTokens?.GetOrDefault("sparkJobNumExecutors", null); if (executorsString != null) { if (!int.TryParse(executorsString, out int executorsInt)) { throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{executorsString}'."); } flowToDeploy.SetObjectToken("sparkJobNumExecutorsInt", executorsInt); } return("no gui config, skipped"); } // Setting TokenName_SparkJobNumExecutors var numExecutorsString = guiConfig?.Process?.JobConfig?.JobNumExecutors; if (!int.TryParse(numExecutorsString, out int numExecutors)) { throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobNumExecutors:'{numExecutorsString}'."); } flowToDeploy.SetObjectToken(TokenName_SparkJobNumExecutors, numExecutors); // Setting TokenName_SparkJobJobExecutorMemory var jobExecutorMemoryString = guiConfig?.Process?.JobConfig?.JobExecutorMemory; if (!int.TryParse(jobExecutorMemoryString, out int jobExecutorMemory)) { throw new ConfigGenerationException($"Invalid value for process.jobconfig.jobExecutorMemory:'{jobExecutorMemoryString}'."); } flowToDeploy.SetStringToken(TokenName_SparkJobJobExecutorMemory, $"{jobExecutorMemory}m"); await Task.Yield(); return("done"); }
private async Task <string> PortSettings(FlowDeploymentSession flowToDeploy) { // TODO: currently this is a whitelist only port a portion of configuration settings // but we might want to port all configuration settings into the token dictionary PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_RuntimeApplicationInsightKey, false); PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_ClusterName, true); PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_ServiceKeyVaultName, true); PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_RuntimeKeyVaultName, true); PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_ConfigFolderHost, false); PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_ConfigFolderContainerPath, false); PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_MetricEventHubConnectionKey, true); PortConfigurationSetting(flowToDeploy, "eventHubResourceGroupName", false); PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_LocalRoot, false); PortConfigurationSetting(flowToDeploy, Constants.ConfigSettingName_LocalMetricsHttpEndpoint, false); await Task.Yield(); return("done"); }
public override async Task <string> Process(FlowDeploymentSession flowToDeploy) { var jobs = flowToDeploy.GetJobs(); if (jobs == null || !jobs.Where(j => j.JobConfigs.Any()).Any()) { return("no jobs, skipped"); } var config = flowToDeploy.Config; var guiConfig = config?.GetGuiConfig(); if (guiConfig == null) { return("no gui input, skipped."); } var projectColumns = guiConfig.Input?.Properties?.NormalizationSnippet?.Trim('\t', ' ', '\r', '\n'); //TODO: make the hardcoded "Raw.*" configurable? var finalProjections = string.IsNullOrEmpty(projectColumns) ? "Raw.*" : projectColumns; var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder); Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder"); var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName); Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName"); var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, "projection.txt"); var savedFile = await RuntimeStorage.SaveFile(filePath, finalProjections); var tokenValue = flowToDeploy.GetTokenString(PrepareProjectionFile.TokenName_ProjectionFiles); var projectionFileSecret = JArray.Parse(tokenValue).FirstOrDefault()?.Value <string>(); if (!string.IsNullOrEmpty(projectionFileSecret)) { await KeyVaultClient.SaveSecretAsync(projectionFileSecret, savedFile); } return("done"); }