/// <summary>
/// This deletes the kernelId as specified
/// </summary>
/// <param name="subscriptionId">subscriptionId that is passed in from the frontend</param>
/// <param name="kernelId">kernelId that needs to be deleted</param>
/// <param name="flowName">flowName</param>
/// <returns>Returns success or failure after the delete kernel api is called</returns>
private async Task <ApiResult> DeleteKernelHelper(string subscriptionId, string kernelId, string flowName)
{
// validate KernelId can't be null
if (string.IsNullOrEmpty(kernelId))
{
_logger.LogError("No Kernel found to delete");
return(ApiResult.CreateError("No Kernel found to delete"));
}
try
{
KernelService kernelService = CreateKernelService(flowName);
var result = await kernelService.DeleteKernelAsync(kernelId);
if (!(result.Error.HasValue && result.Error.Value))
{
//Passing in false as the last parameter since this is a delete call and the entry needs to be deleted from the blob
await kernelService.UpdateGarbageCollectKernelBlob(kernelId, "", "", _engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName), false);
}
return(result);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
return(ApiResult.CreateError(ex.Message));
}
}
/// <summary>
/// CreateAndInitializeKernelHelper is the helper that does the heavy listing for creating and initializing the kernel
/// </summary>
/// <param name="rawSchema">rawSchema as passed in from the frontend</param>
/// <param name="userId">userId as passed in from the frontend</param>
/// <param name="flowId">flowId as passed in from the frontend</param>
/// <param name="sampleDataPath">sampleDataPath where the sample data is stored</param>
/// <param name="normalizationSnippet">normalizationSnippet as passed in from the frontend</param>
/// <param name="referenceDatas">referenceDatas as passed in from the frontend</param>
/// <param name="functions">functions as passed in from the frontend</param>
/// <returns></returns>
private async Task <ApiResult> CreateAndInitializeKernelHelper(string rawSchema, string userId, string flowId, string sampleDataPath, string normalizationSnippet, List <ReferenceDataObject> referenceDatas, List <FunctionObject> functions)
{
try
{
//Create the xml with the scala steps to execute to initialize the kernel
DiagnosticInputhelper(rawSchema, sampleDataPath, normalizationSnippet, flowId);
KernelService kernelService = CreateKernelService(flowId);
var response = await kernelService.GarbageCollectListOfKernels(_engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName));
response = await kernelService.CreateKernelAsync();
if (response.Error.HasValue && response.Error.Value)
{
return(ApiResult.CreateError(response.Message));
}
string kernelId = response.Result.ToString();
response = await kernelService.UpdateGarbageCollectKernelBlob(kernelId, userId, flowId, _engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName), true);
if (response.Error.HasValue && response.Error.Value)
{
await kernelService.DeleteKernelAsync(kernelId);
return(ApiResult.CreateError(response.Message));
}
if (_engineEnvironment.EngineFlowConfig.SparkType == Config.ConfigDataModel.Constants.SparkTypeDataBricks)
{
kernelService.MountStorage(_engineEnvironment.EngineFlowConfig.OpsStorageAccountName, _engineEnvironment.EngineFlowConfig.SparkKeyVaultName, sampleDataPath, kernelId);
}
response = await kernelService.CreateandInitializeKernelAsync(kernelId, SetupSteps, false, referenceDatas, functions);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
else
{
return(ApiResult.CreateSuccess(response.Result));
}
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
return(ApiResult.CreateError(ex.Message));
}
}
/// <summary>
/// This is the helper function that does the heavy lifting for refreshing the kernel
/// </summary>
/// <param name="diag">InteractiveQueryObject that gets passed in from frontend</param>
/// <param name="isReSample">This function is also used for refreshing the sample as well. This bool decides what action to take - use the existing kernel or create a new one</param>
/// <returns>>Returns the KernelId plus a warning in case the normalization snippet contains columns that are not part of the schema as generated on success. Returns Error in case of error.</returns>
public async Task <ApiResult> RecycleKernelHelper(InteractiveQueryObject diag, bool isReSample)
{
try
{
ApiResult response;
if (_engineEnvironment.EngineFlowConfig == null)
{
response = await _engineEnvironment.GetEnvironmentVariables();
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
}
if (string.IsNullOrEmpty(diag.Name))
{
diag.Name = await _engineEnvironment.GetUniqueName(Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId), diag.DisplayName);
}
KernelService kernelService = CreateKernelService(diag.Name);
//Create the xml with the scala steps to execute to initialize the kernel
var hashValue = Helper.GetHashCode(diag.UserName);
var sampleDataPath = Helper.SetValueBasedOnSparkType(_engineEnvironment.EngineFlowConfig.SparkType,
Path.Combine(_engineEnvironment.OpsSparkSamplePath, $"{diag.Name}-{hashValue}.json"),
Helper.ConvertToDbfsFilePath(_engineEnvironment.OpsSparkSamplePath, $"{diag.Name}-{hashValue}.json"));
DiagnosticInputhelper(diag.InputSchema, sampleDataPath, diag.NormalizationSnippet, diag.Name);
response = await kernelService.RecycleKernelAsync(diag.KernelId, diag.UserName, diag.Name, _engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName), SetupSteps, isReSample, diag.ReferenceDatas, diag.Functions);
return(response);
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
return(ApiResult.CreateError(ex.Message));
}
}
/// <summary>
/// ExecuteQueryHelper is the function that gets called for executing the user code
/// </summary>
/// <param name="jObject">jObject as passed in from the frontend</param>
/// <returns>Returns the result from the kernel after executing the query</returns>
private async Task <ApiResult> ExecuteQueryHelper(JObject jObject)
{
// validate the Execute Query API parameters i.e. the query object
if (jObject == null)
{
_logger.LogError("Query Object is null");
return(ApiResult.CreateError("Query Object is null"));
}
var query = jObject.ToObject <InteractiveQueryObject>();
ApiResult response = await _engineEnvironment.GetEnvironmentVariables();
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
if (string.IsNullOrEmpty(Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId)))
{
return(ApiResult.CreateError("Subscription can't be null"));
}
try
{
KernelService kernelService = CreateKernelService(query.Name);
var result = await kernelService.ExecuteQueryAsync(query.Query, query.KernelId);
return(result);
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
return(ApiResult.CreateError(ex.Message));
}
}
/// <summary>
/// GetSampleInputFromQueryHelper gets the sample from the input table in the user query
/// </summary>
/// <param name="jObject">jObject </param>
/// <returns>Returns the sample result from the input from the user query being executed</returns>
private async Task <ApiResult> GetSampleInputFromQueryHelper(JObject jObject)
{
// validate JObject which is the query object in this case - is not null or empty
if (jObject == null)
{
_logger.LogError("Query Object is null");
return(ApiResult.CreateError("Query Object is null"));
}
var query = jObject.ToObject <InteractiveQueryObject>();
try
{
KernelService kernelService = CreateKernelService(query.Name);
var result = await kernelService.GetSampleInputFromQueryAsync(query.Query, query.KernelId);
return(result);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
return(ApiResult.CreateError(ex.Message));
}
}
/// <summary>
/// This is the API method that gets called from the front end on a regular cadence and will delete all the kernels that were created
/// </summary>
/// <param name="flowName">flowName</param>
/// <returns>Returns the result whether the list of kernels were deleted or not. We don't fail if one of the kernels fails to delete because it just does not exist</returns>
public async Task <ApiResult> DeleteAllKernels(string flowName)
{
var response = await _engineEnvironment.GetEnvironmentVariables().ConfigureAwait(false);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
KernelService kernelService = CreateKernelService(flowName);
_logger.LogInformation("Deleting all Kernels...");
response = await kernelService.GarbageCollectListOfKernels(_engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName), true).ConfigureAwait(false);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
var result = response.Result.ToString();
_logger.LogInformation("Deleted and cleared all Kernels successfully");
return(ApiResult.CreateSuccess(result));
}
/// <summary>
/// This is the API method that gets called from the front end on a regular cadence and will delete all the kernels that are more than 3 hours old
/// </summary>
/// <param name="flowName">flowName</param>
/// <returns>Returns the result whether the list of kernels were deleted or not. We don't fail if one of the kernels fails to delete because it just does not exist</returns>
public async Task <ApiResult> DeleteKernels(string flowName)
{
var response = await _engineEnvironment.GetEnvironmentVariables().ConfigureAwait(false);
var subscriptionId = Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
KernelService kernelService = CreateKernelService(flowName);
response = await kernelService.GarbageCollectListOfKernels(_engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName)).ConfigureAwait(false);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
var result = response.Result.ToString();
return(ApiResult.CreateSuccess(result));
}
/// <summary>
/// GetSampleInputFromQueryHelper gets the sample from the input table in the user query
/// </summary>
/// <param name="jObject">jObject </param>
/// <returns>Returns the sample result from the input from the user query being executed</returns>
private async Task <ApiResult> GetSampleInputFromQueryHelper(JObject jObject)
{
// validate JObject which is the query object in this case - is not null or empty
if (jObject == null)
{
_logger.LogError("Query Object is null");
return(ApiResult.CreateError("Query Object is null"));
}
var query = jObject.ToObject <InteractiveQueryObject>();
try
{
KernelService kernelService = new KernelService(_engineEnvironment.EngineFlowConfig.JobURLBase, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix, _engineEnvironment.SparkConnInfo, _logger);
var result = await kernelService.GetSampleInputFromQueryAsync(query.Query, query.KernelId);
return(result);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
return(ApiResult.CreateError(ex.Message));
}
}