public async Task <ApiResult> GetInputSchema(JObject jObject)
{
var diag = jObject.ToObject <InteractiveQueryObject>();
var eventHubNames = Helper.ParseEventHubNames(diag.EventhubNames);
if (_engineEnvironment.EngineFlowConfig == null)
{
await _engineEnvironment.GetEnvironmentVariables();
}
if (string.IsNullOrEmpty(diag.Name))
{
diag.Name = await _engineEnvironment.GetUniqueName(Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId), diag.DisplayName);
}
var connectionString = Helper.GetSecretFromKeyvaultIfNeeded(diag.EventhubConnectionString);
if (diag.InputType == Constants.InputType_EventHub)
{
string ehName = Helper.ParseEventHub(connectionString);
eventHubNames = new List <string>()
{
ehName
};
}
string eventHubNamespace = Helper.ParseEventHubNamespace(connectionString);
//Fixing a bug where '_' checkpoint container name cannot have '_'
var checkPointContainerName = $"{_engineEnvironment.CheckPointContainerNameHelper(diag.Name)}-checkpoints";
// ResourceCreation is one of the environment variables.
// If you don't want to create resource, you can set this to false.
if (_engineEnvironment.ResourceCreation && (diag.InputType == Constants.InputType_EventHub || diag.InputType == Constants.InputType_IoTHub))
{
var inputSubscriptionId = string.IsNullOrEmpty(diag.InputSubscriptionId) ? Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId) : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputSubscriptionId);
var inputResourceGroup = string.IsNullOrEmpty(diag.InputResourceGroup) ? _engineEnvironment.EngineFlowConfig.EventHubResourceGroupName : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputResourceGroup);
// Create consumer group if it doesn't exist
foreach (string ehName in eventHubNames)
{
var result = EventHub.CreateConsumerGroup(inputSubscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, inputResourceGroup, _engineEnvironment.EngineFlowConfig.EventHubResourceGroupLocation, eventHubNamespace, ehName, _engineEnvironment.EngineFlowConfig.ConsumerGroup, diag.InputType, _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
if (result.Error.HasValue && result.Error.Value)
{
_logger.LogError(result.Message);
return(ApiResult.CreateError(result.Message));
}
}
}
var bootstrapServers = Helper.TryGetBootstrapServers(connectionString);
// Sample events and generate schema
SchemaGenerator sg = new SchemaGenerator(bootstrapServers, connectionString, eventHubNames, _engineEnvironment.EngineFlowConfig.ConsumerGroup, _engineEnvironment.OpsBlobConnectionString, checkPointContainerName, _engineEnvironment.EngineFlowConfig.OpsBlobDirectory, diag.InputType, diag.InputMode, diag.BatchInputs, _logger);
SchemaResult schema = await sg.GetSchemaAsync(_engineEnvironment.OpsBlobConnectionString, OpsSamplePath, diag.UserName, diag.Name, diag.Seconds);
return(ApiResult.CreateSuccess(JObject.FromObject(schema)));
}
public async Task <ApiResult> RefreshInputDataAndKernel(JObject jObject)
{
var diag = jObject.ToObject <InteractiveQueryObject>();
var response = await _engineEnvironment.GetEnvironmentVariables();
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
//Refresh the sample data
SchemaInferenceManager sim = new SchemaInferenceManager(_logger);
response = await sim.RefreshSample(jObject);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
InteractiveQueryManager iqm = new InteractiveQueryManager(_logger);
response = await iqm.RecycleKernelHelper(diag, true);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
var result = response.Result;
_logger.LogInformation("RefreshInputDataAndKernel successful!");
return(ApiResult.CreateSuccess(result));
}
private async Task <Dictionary <string, string> > SetRequestHeader()
{
var response = await _engineEnvironment.GetEnvironmentVariables().ConfigureAwait(false);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
throw new Exception("Can't get environment variables.");
}
var clientId = _engineEnvironment.EngineFlowConfig.ConfiggenClientId;
var clientSecret = Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.ConfiggenClientSecret);
var clientResourceId = Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.ConfiggenClientResourceId);
var tenantId = _engineEnvironment.EngineFlowConfig.ConfiggenTenantId;
var authenticationContext = new AuthenticationContext($"https://login.windows.net/{tenantId}");
var credential = new ClientCredential(clientId, clientSecret);
var apiToken = authenticationContext.AcquireTokenAsync(clientResourceId, credential).Result.AccessToken;
var jwtHandler = new JwtSecurityTokenHandler();
var readableToken = jwtHandler.CanReadToken(apiToken);
Dictionary <string, string> headers = new Dictionary <string, string>();
if (readableToken == true)
{
var token = jwtHandler.ReadJwtToken(apiToken);
var roleValue = token.Claims.FirstOrDefault(c => c.Type == "roles")?.Value;
if (!string.IsNullOrEmpty(roleValue))
{
headers.Add(Constants.UserRolesHeader, roleValue);
}
}
return(headers);
}
/// <summary>
/// This method gets called for api/kernel
/// </summary>
/// <param name="jObject">JObject gets passed from the frontend containing the required parameters for this method</param>
/// <returns>Returns the KernelId plus a warning in case the normalization snippet contains columns that are not part of the schema as generated on success. Returns Error in case of error.</returns>
public async Task <ApiResult> CreateAndInitializeKernel(JObject jObject)
{
var diag = jObject.ToObject <InteractiveQueryObject>();
ApiResult response;
if (_engineEnvironment.EngineFlowConfig == null)
{
response = await _engineEnvironment.GetEnvironmentVariables();
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
}
if (string.IsNullOrEmpty(diag.Name))
{
diag.Name = await _engineEnvironment.GetUniqueName(Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId), diag.DisplayName);
}
var hashValue = Helper.GetHashCode(diag.UserName);
string sampleDataPath = Helper.SetValueBasedOnSparkType(_engineEnvironment.EngineFlowConfig.SparkType,
Path.Combine(_engineEnvironment.OpsSparkSamplePath, $"{diag.Name}-{hashValue}.json"),
Helper.ConvertToDbfsFilePath(_engineEnvironment.OpsSparkSamplePath, $"{diag.Name}-{hashValue}.json"));
response = await CreateAndInitializeKernelHelper(diag.InputSchema, diag.UserName, diag.Name, sampleDataPath, diag.NormalizationSnippet, diag.ReferenceDatas, diag.Functions);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
var result = response.Result;
return(ApiResult.CreateSuccess(result));
}
/// This is the function that can be called for deleting all assets created on save of a flow: consumergroup, secrets in Key Vault, cosmosDB products document such that this flow stops showing up in the UI under Flows and blobs
/// Please note if a a new asset is created on Azure as part of saving the flow or any other action taken by user in the UI, this function will need to be updated so that the asset can be deleted on delete of the flow.
/// </summary>
/// <param name="jObject">jObject requires: Subscription; Name; EventhubConnectionString; IsIotHub;EventhubName;userID</param>
/// <returns>Returns result - success or failure as the case maybe</returns>
public async Task <ApiResult> DeleteFlow(JObject jObject)
{
var diag = jObject.ToObject <InteractiveQueryObject>();
ConfigName = diag.Name;
bool errorExists = false;
var response = await _engineEnvironment.GetEnvironmentVariables();
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
///Delete consumer group
_logger.LogInformation($"For FlowId: {ConfigName} Deleting flow specific consumer group.. ");
var inputEventhubConnection = Helper.GetSecretFromKeyvaultIfNeeded(diag.EventhubConnectionString);
_inputEventhubConnectionStringRef = Helper.IsKeyVault(diag.EventhubConnectionString) ? diag.EventhubConnectionString : Helper.GenerateNewSecret(_keySecretList, _engineEnvironment.EngineFlowConfig.SparkKeyVaultName, ConfigName + "-input-eventhubconnectionstring", diag.EventhubConnectionString, false);
diag.EventhubConnectionString = _inputEventhubConnectionStringRef;
if (diag.InputType == Constants.InputType_EventHub)
{
var ehName = Helper.ParseEventHub(inputEventhubConnection);
_eventHubNamespace = Helper.ParseEventHubNamespace(inputEventhubConnection);
_eventHubNameRole = Helper.ParseEventHubPolicyName(inputEventhubConnection);
_eventHubPrimaryKeyListener = Helper.ParseEventHubAccessKey(inputEventhubConnection);
if (string.IsNullOrWhiteSpace(ehName) || string.IsNullOrWhiteSpace(_eventHubNamespace) || string.IsNullOrWhiteSpace(_eventHubNameRole) || string.IsNullOrWhiteSpace(_eventHubPrimaryKeyListener))
{
string error = "The connection string for Event Hub input type must contain Endpoint, SharedAccessKeyName, SharedAccessKey, and EntityPath";
_logger.LogError(error);
errorExists = true;
}
_eventHubNames = new List <string>()
{
ehName
};
}
else
{
_eventHubNames = Helper.ParseEventHubNames(diag.EventhubNames);
_eventHubNamespace = Helper.ParseEventHubNamespace(inputEventhubConnection);
_eventHubNameRole = Helper.ParseEventHubPolicyName(inputEventhubConnection);
_eventHubPrimaryKeyListener = Helper.ParseEventHubAccessKey(inputEventhubConnection);
if (_eventHubNames.Count < 1)
{
string error = "The event hub-compatible name for IoT Hub input type must be defined";
_logger.LogError(error);
errorExists = true;
}
if (string.IsNullOrWhiteSpace(_eventHubNamespace) || string.IsNullOrWhiteSpace(_eventHubNameRole) || string.IsNullOrWhiteSpace(_eventHubPrimaryKeyListener))
{
string error = "The event hub-compatible endpoint for IoT Hub input type must contain Endpoint, SharedAccessKeyName, and SharedAccessKey";
_logger.LogError(error);
errorExists = true;
}
}
// ResourceCreation is one of the environment variables.
// If you don't want to create resource, you can set this to false.
if (_engineEnvironment.ResourceCreation && (diag.InputType == Constants.InputType_EventHub || diag.InputType == Constants.InputType_IoTHub))
{
var inputSubscriptionId = string.IsNullOrEmpty(diag.InputSubscriptionId) ? Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId) : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputSubscriptionId);
var inputResourceGroup = string.IsNullOrEmpty(diag.InputResourceGroup) ? _engineEnvironment.EngineFlowConfig.EventHubResourceGroupName : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputResourceGroup);
foreach (string ehName in _eventHubNames)
{
var result = EventHub.DeleteConsumerGroup(inputSubscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, inputResourceGroup, _engineEnvironment.EngineFlowConfig.EventHubResourceGroupLocation, _eventHubNamespace, ehName, ConsumerGroupName, diag.InputType, _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
if (result.Error.HasValue && result.Error.Value)
{
_logger.LogError(result.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully deleted flow specific consumer group");
}
}
}
///Delete cosmosDB document related to a flow
response = await CosmosDB.DeleteConfigFromDocumentDB(_engineEnvironment.CosmosDBDatabaseName, _engineEnvironment.CosmosDBEndPoint, _engineEnvironment.CosmosDBUserName, _engineEnvironment.CosmosDBPassword, "flows", ConfigName);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific cosmosDB entry");
}
///Delete configs stored in blobs
// ruleDefinitions
response = await BlobHelper.DeleteBlob(_engineEnvironment.FlowBlobConnectionString, Path.Combine(RuleDefinitionPath, RuleDefinitionFileName));
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific rules definition blob");
}
// outputTemplates
response = await BlobHelper.DeleteBlob(_engineEnvironment.FlowBlobConnectionString, Path.Combine(OutputTemplatePath, OutputTemplateFileName));
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific output template blob");
}
string resourceGroupLocation = _engineEnvironment.EngineFlowConfig.ResourceGroupLocation;
string resourceGroupName = _engineEnvironment.EngineFlowConfig.ResourceGroupName;
string storageAccountName = _engineEnvironment.EngineFlowConfig.StorageAccountName;
string containerPath = Path.Combine(_flowContainerName, _engineEnvironment.EngineFlowConfig.EnvironmentType, ConfigName);
string subscriptionId = Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId);
BlobStorage.DeleteAllConfigsFromBlobStorage(subscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, resourceGroupName, resourceGroupLocation, storageAccountName, _Centralprocessing, Path.Combine(_engineEnvironment.EngineFlowConfig.ContainerPath, ConfigName), _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific blobs under the folder {ConfigName} under container {_Centralprocessing}");
BlobStorage.DeleteAllConfigsFromBlobStorage(subscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, resourceGroupName, resourceGroupLocation, storageAccountName, _Centralprocessing, Path.Combine(_engineEnvironment.EngineFlowConfig.ContainerPath, ConfigName), _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
BlobStorage.DeleteAllConfigsFromBlobStorage(subscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, resourceGroupName, resourceGroupLocation, storageAccountName, _flowContainerName, Path.Combine(_engineEnvironment.EngineFlowConfig.EnvironmentType, ConfigName), _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific productconfig: {ProductConfigName} and {JobConfigName} for {ConfigName}.");
/// Delete sample data and the checkpoints folder if it exists for that flow
var hashValue = Helper.GetHashCode(diag.UserName);
await BlobHelper.DeleteBlob(_engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsSamplePath, $"{ConfigName}-{hashValue}.json"));
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific sampledata file: {ConfigName}-{ hashValue}.json");
await BlobHelper.DeleteAllBlobsInAContainer(_engineEnvironment.OpsBlobConnectionString, $"{_engineEnvironment.CheckPointContainerNameHelper(ConfigName)}-checkpoints", _engineEnvironment.EngineFlowConfig.OpsBlobDirectory);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific checkpoints for {ConfigName}.");
_logger.LogInformation("Deleting flow specific secrets..");
///Delete secrets specific to a flow from KeyVault
KeyVault.GetSecretsAndDeleteFromKeyvault(_engineEnvironment.EngineFlowConfig.SparkKeyVaultName, ConfigName);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific secrets");
if (!errorExists)
{
return(ApiResult.CreateSuccess("Deleted!"));
}
else
{
return(ApiResult.CreateError("Deleted but with some error. Please check logs for details"));
}
}
