private async Task <FlowBlobOutputSpec> ProcessOutputBlob(string configName, FlowGuiOutput uiOutput) { if (uiOutput != null && uiOutput.Properties != null) { var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName]; string connectionString = await KeyVaultClient.GetSecretFromKeyVaultAsync(sparkKeyVaultName, uiOutput.Properties.ConnectionString); var blobPath = $"wasbs://{uiOutput.Properties.ContainerName}{ParseBlobAccountName(connectionString)}.blob.core.windows.net/{uiOutput.Properties.BlobPrefix}/%1$tY/%1$tm/%1$td/%1$tH/${{quarterBucket}}/${{minuteBucket}}"; var secretId = $"{configName}-output"; var blobPathSecret = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, secretId, blobPath, true); FlowBlobOutputSpec blobOutput = new FlowBlobOutputSpec() { CompressionType = uiOutput.Properties.CompressionType, Format = uiOutput.Properties.Format, Groups = new BlobOutputGroups() { Main = new BlobOutputMain() { Folder = blobPathSecret } } }; return(blobOutput); } else { return(null); } }
private FlowBlobOutputSpec ProcessOutputLocal(string configName, FlowGuiOutput uiOutput) { if (uiOutput != null && uiOutput.Properties != null) { string outputPath = uiOutput.Properties.ConnectionString.TrimEnd('/'); var localPath = $"{outputPath}/%1$tY/%1$tm/%1$td/%1$tH/${{quarterBucket}}/${{minuteBucket}}"; FlowBlobOutputSpec localOutput = new FlowBlobOutputSpec() { CompressionType = uiOutput.Properties.CompressionType, Format = uiOutput.Properties.Format, Groups = new BlobOutputGroups() { Main = new BlobOutputMain() { Folder = localPath } } }; return(localOutput); } else { return(null); } }
private FlowEventHubOutputSpec ProcessOutputEventHub(FlowGuiOutput uiOutput) { if (uiOutput != null && uiOutput.Properties != null) { FlowEventHubOutputSpec eventhubOutput = new FlowEventHubOutputSpec() { ConnectionStringRef = uiOutput.Properties.ConnectionString, CompressionType = uiOutput.Properties.CompressionType, Format = uiOutput.Properties.Format }; return(eventhubOutput); } else { return(null); } }
private FlowCosmosDbOutputSpec ProcessOutputCosmosDb(FlowGuiOutput uiOutput) { if (uiOutput != null && uiOutput.Properties != null) { FlowCosmosDbOutputSpec cosmoDbOutput = new FlowCosmosDbOutputSpec() { ConnectionStringRef = uiOutput.Properties.ConnectionString, Database = uiOutput.Properties.Db, Collection = uiOutput.Properties.Collection }; return(cosmoDbOutput); } else { return(null); } }
private FlowEventHubOutputSpec ProcessOutputMetric(FlowGuiOutput uiOutput) { if (uiOutput != null && uiOutput.Properties != null) { var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName]; var metricsEhConnectionStringKey = Configuration[Constants.ConfigSettingName_MetricEventHubConnectionKey]; FlowEventHubOutputSpec eventhubOutput = new FlowEventHubOutputSpec() { ConnectionStringRef = KeyVaultUri.ComposeUri(sparkKeyVaultName, metricsEhConnectionStringKey), CompressionType = "none", Format = "json" }; return(eventhubOutput); } else { return(null); } }
private async Task <FlowSqlOutputSpec> ProcessOutputSql(string configName, FlowGuiOutput uiOutput) { if (uiOutput != null && uiOutput.Properties != null) { var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName]; string connectionString = await KeyVaultClient.ResolveSecretUriAsync(uiOutput.Properties.ConnectionString).ConfigureAwait(false); var database = GetValueFromJdbcConnection(connectionString, "database"); var user = GetValueFromJdbcConnection(connectionString, "user"); var pwd = GetValueFromJdbcConnection(connectionString, "password"); var url = GetUrlFromJdbcConnection(connectionString); var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null; // Save password and url in keyvault var pwdSecretId = $"{configName}-outSqlPassword"; var pwdRef = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, pwdSecretId, pwd, sparkType, true).ConfigureAwait(false); var urlSecretId = $"{configName}-outSqlUrl"; var urlRef = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, urlSecretId, url, sparkType, true).ConfigureAwait(false); FlowSqlOutputSpec sqlOutput = new FlowSqlOutputSpec() { ConnectionStringRef = uiOutput.Properties.ConnectionString, TableName = uiOutput.Properties.TableName, WriteMode = uiOutput.Properties.WriteMode, UseBulkInsert = uiOutput.Properties.UseBulkInsert, DatabaseName = database, User = user, Password = pwdRef, Url = urlRef }; return(sqlOutput); } else { return(null); } }
private async Task <FlowBlobOutputSpec> ProcessOutputBlob(string inputMode, string configName, FlowGuiOutput uiOutput) { if (uiOutput != null && uiOutput.Properties != null) { var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName]; string connectionString = await KeyVaultClient.ResolveSecretUriAsync(uiOutput.Properties.ConnectionString); var accountName = ConfigHelper.ParseBlobAccountName(connectionString); var timeFormat = inputMode != Constants.InputMode_Batching ? $"%1$tY/%1$tm/%1$td/%1$tH/${{quarterBucket}}/${{minuteBucket}}" : $"{TransformPartitionFormat(uiOutput.Properties.BlobPartitionFormat)}"; var blobPath = $"wasbs://{uiOutput.Properties.ContainerName}@{accountName}.blob.core.windows.net/{uiOutput.Properties.BlobPrefix}/{timeFormat}"; var secretId = $"{configName}-output"; var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null; var blobPathSecret = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, secretId, blobPath, sparkType, true); await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, $"{Constants.AccountSecretPrefix}{accountName}", ConfigHelper.ParseBlobAccountKey(connectionString), sparkType, false); FlowBlobOutputSpec blobOutput = new FlowBlobOutputSpec() { CompressionType = uiOutput.Properties.CompressionType, Format = uiOutput.Properties.Format, Groups = new BlobOutputGroups() { Main = new BlobOutputMain() { Folder = blobPathSecret } } }; return(blobOutput); } else { return(null); } }