private async Task <FlowBlobOutputSpec> ProcessOutputBlob(string configName, FlowGuiOutput uiOutput)
{
if (uiOutput != null && uiOutput.Properties != null)
{
var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
string connectionString = await KeyVaultClient.GetSecretFromKeyVaultAsync(sparkKeyVaultName, uiOutput.Properties.ConnectionString);
var blobPath = $"wasbs://{uiOutput.Properties.ContainerName}{ParseBlobAccountName(connectionString)}.blob.core.windows.net/{uiOutput.Properties.BlobPrefix}/%1$tY/%1$tm/%1$td/%1$tH/${{quarterBucket}}/${{minuteBucket}}";
var secretId = $"{configName}-output";
var blobPathSecret = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, secretId, blobPath, true);
FlowBlobOutputSpec blobOutput = new FlowBlobOutputSpec()
{
CompressionType = uiOutput.Properties.CompressionType,
Format = uiOutput.Properties.Format,
Groups = new BlobOutputGroups()
{
Main = new BlobOutputMain()
{
Folder = blobPathSecret
}
}
};
return(blobOutput);
}
else
{
return(null);
}
}
private FlowBlobOutputSpec ProcessOutputLocal(string configName, FlowGuiOutput uiOutput)
{
if (uiOutput != null && uiOutput.Properties != null)
{
string outputPath = uiOutput.Properties.ConnectionString.TrimEnd('/');
var localPath = $"{outputPath}/%1$tY/%1$tm/%1$td/%1$tH/${{quarterBucket}}/${{minuteBucket}}";
FlowBlobOutputSpec localOutput = new FlowBlobOutputSpec()
{
CompressionType = uiOutput.Properties.CompressionType,
Format = uiOutput.Properties.Format,
Groups = new BlobOutputGroups()
{
Main = new BlobOutputMain()
{
Folder = localPath
}
}
};
return(localOutput);
}
else
{
return(null);
}
}
private FlowEventHubOutputSpec ProcessOutputEventHub(FlowGuiOutput uiOutput)
{
if (uiOutput != null && uiOutput.Properties != null)
{
FlowEventHubOutputSpec eventhubOutput = new FlowEventHubOutputSpec()
{
ConnectionStringRef = uiOutput.Properties.ConnectionString,
CompressionType = uiOutput.Properties.CompressionType,
Format = uiOutput.Properties.Format
};
return(eventhubOutput);
}
else
{
return(null);
}
}
private FlowCosmosDbOutputSpec ProcessOutputCosmosDb(FlowGuiOutput uiOutput)
{
if (uiOutput != null && uiOutput.Properties != null)
{
FlowCosmosDbOutputSpec cosmoDbOutput = new FlowCosmosDbOutputSpec()
{
ConnectionStringRef = uiOutput.Properties.ConnectionString,
Database = uiOutput.Properties.Db,
Collection = uiOutput.Properties.Collection
};
return(cosmoDbOutput);
}
else
{
return(null);
}
}
private FlowEventHubOutputSpec ProcessOutputMetric(FlowGuiOutput uiOutput)
{
if (uiOutput != null && uiOutput.Properties != null)
{
var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
var metricsEhConnectionStringKey = Configuration[Constants.ConfigSettingName_MetricEventHubConnectionKey];
FlowEventHubOutputSpec eventhubOutput = new FlowEventHubOutputSpec()
{
ConnectionStringRef = KeyVaultUri.ComposeUri(sparkKeyVaultName, metricsEhConnectionStringKey),
CompressionType = "none",
Format = "json"
};
return(eventhubOutput);
}
else
{
return(null);
}
}
private async Task <FlowSqlOutputSpec> ProcessOutputSql(string configName, FlowGuiOutput uiOutput)
{
if (uiOutput != null && uiOutput.Properties != null)
{
var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
string connectionString = await KeyVaultClient.ResolveSecretUriAsync(uiOutput.Properties.ConnectionString).ConfigureAwait(false);
var database = GetValueFromJdbcConnection(connectionString, "database");
var user = GetValueFromJdbcConnection(connectionString, "user");
var pwd = GetValueFromJdbcConnection(connectionString, "password");
var url = GetUrlFromJdbcConnection(connectionString);
var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;
// Save password and url in keyvault
var pwdSecretId = $"{configName}-outSqlPassword";
var pwdRef = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, pwdSecretId, pwd, sparkType, true).ConfigureAwait(false);
var urlSecretId = $"{configName}-outSqlUrl";
var urlRef = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, urlSecretId, url, sparkType, true).ConfigureAwait(false);
FlowSqlOutputSpec sqlOutput = new FlowSqlOutputSpec()
{
ConnectionStringRef = uiOutput.Properties.ConnectionString,
TableName = uiOutput.Properties.TableName,
WriteMode = uiOutput.Properties.WriteMode,
UseBulkInsert = uiOutput.Properties.UseBulkInsert,
DatabaseName = database,
User = user,
Password = pwdRef,
Url = urlRef
};
return(sqlOutput);
}
else
{
return(null);
}
}
private async Task <FlowBlobOutputSpec> ProcessOutputBlob(string inputMode, string configName, FlowGuiOutput uiOutput)
{
if (uiOutput != null && uiOutput.Properties != null)
{
var sparkKeyVaultName = Configuration[Constants.ConfigSettingName_RuntimeKeyVaultName];
string connectionString = await KeyVaultClient.ResolveSecretUriAsync(uiOutput.Properties.ConnectionString);
var accountName = ConfigHelper.ParseBlobAccountName(connectionString);
var timeFormat = inputMode != Constants.InputMode_Batching ? $"%1$tY/%1$tm/%1$td/%1$tH/${{quarterBucket}}/${{minuteBucket}}" : $"{TransformPartitionFormat(uiOutput.Properties.BlobPartitionFormat)}";
var blobPath = $"wasbs://{uiOutput.Properties.ContainerName}@{accountName}.blob.core.windows.net/{uiOutput.Properties.BlobPrefix}/{timeFormat}";
var secretId = $"{configName}-output";
var sparkType = Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string value) ? value : null;
var blobPathSecret = await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, secretId, blobPath, sparkType, true);
await KeyVaultClient.SaveSecretAsync(sparkKeyVaultName, $"{Constants.AccountSecretPrefix}{accountName}", ConfigHelper.ParseBlobAccountKey(connectionString), sparkType, false);
FlowBlobOutputSpec blobOutput = new FlowBlobOutputSpec()
{
CompressionType = uiOutput.Properties.CompressionType,
Format = uiOutput.Properties.Format,
Groups = new BlobOutputGroups()
{
Main = new BlobOutputMain()
{
Folder = blobPathSecret
}
}
};
return(blobOutput);
}
else
{
return(null);
}
}
