/// <summary> /// This deletes the kernelId as specified /// </summary> /// <param name="subscriptionId">subscriptionId that is passed in from the frontend</param> /// <param name="kernelId">kernelId that needs to be deleted</param> /// <param name="flowName">flowName</param> /// <returns>Returns success or failure after the delete kernel api is called</returns> private async Task <ApiResult> DeleteKernelHelper(string subscriptionId, string kernelId, string flowName) { // validate KernelId can't be null if (string.IsNullOrEmpty(kernelId)) { _logger.LogError("No Kernel found to delete"); return(ApiResult.CreateError("No Kernel found to delete")); } try { KernelService kernelService = CreateKernelService(flowName); var result = await kernelService.DeleteKernelAsync(kernelId); if (!(result.Error.HasValue && result.Error.Value)) { //Passing in false as the last parameter since this is a delete call and the entry needs to be deleted from the blob await kernelService.UpdateGarbageCollectKernelBlob(kernelId, "", "", _engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName), false); } return(result); } catch (Exception ex) { _logger.LogError(ex.Message); return(ApiResult.CreateError(ex.Message)); } }
/// <summary> /// CreateAndInitializeKernelHelper is the helper that does the heavy listing for creating and initializing the kernel /// </summary> /// <param name="rawSchema">rawSchema as passed in from the frontend</param> /// <param name="userId">userId as passed in from the frontend</param> /// <param name="flowId">flowId as passed in from the frontend</param> /// <param name="sampleDataPath">sampleDataPath where the sample data is stored</param> /// <param name="normalizationSnippet">normalizationSnippet as passed in from the frontend</param> /// <param name="referenceDatas">referenceDatas as passed in from the frontend</param> /// <param name="functions">functions as passed in from the frontend</param> /// <returns></returns> private async Task <ApiResult> CreateAndInitializeKernelHelper(string rawSchema, string userId, string flowId, string sampleDataPath, string normalizationSnippet, List <ReferenceDataObject> referenceDatas, List <FunctionObject> functions) { try { //Create the xml with the scala steps to execute to initialize the kernel DiagnosticInputhelper(rawSchema, sampleDataPath, normalizationSnippet, flowId); KernelService kernelService = CreateKernelService(flowId); var response = await kernelService.GarbageCollectListOfKernels(_engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName)); response = await kernelService.CreateKernelAsync(); if (response.Error.HasValue && response.Error.Value) { return(ApiResult.CreateError(response.Message)); } string kernelId = response.Result.ToString(); response = await kernelService.UpdateGarbageCollectKernelBlob(kernelId, userId, flowId, _engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName), true); if (response.Error.HasValue && response.Error.Value) { await kernelService.DeleteKernelAsync(kernelId); return(ApiResult.CreateError(response.Message)); } if (_engineEnvironment.EngineFlowConfig.SparkType == Config.ConfigDataModel.Constants.SparkTypeDataBricks) { kernelService.MountStorage(_engineEnvironment.EngineFlowConfig.OpsStorageAccountName, _engineEnvironment.EngineFlowConfig.SparkKeyVaultName, sampleDataPath, kernelId); } response = await kernelService.CreateandInitializeKernelAsync(kernelId, SetupSteps, false, referenceDatas, functions); if (response.Error.HasValue && response.Error.Value) { _logger.LogError(response.Message); return(ApiResult.CreateError(response.Message)); } else { return(ApiResult.CreateSuccess(response.Result)); } } catch (Exception ex) { _logger.LogError(ex, ex.Message); return(ApiResult.CreateError(ex.Message)); } }
/// <summary> /// This is the helper function that does the heavy lifting for refreshing the kernel /// </summary> /// <param name="diag">InteractiveQueryObject that gets passed in from frontend</param> /// <param name="isReSample">This function is also used for refreshing the sample as well. This bool decides what action to take - use the existing kernel or create a new one</param> /// <returns>>Returns the KernelId plus a warning in case the normalization snippet contains columns that are not part of the schema as generated on success. Returns Error in case of error.</returns> public async Task <ApiResult> RecycleKernelHelper(InteractiveQueryObject diag, bool isReSample) { try { ApiResult response; if (_engineEnvironment.EngineFlowConfig == null) { response = await _engineEnvironment.GetEnvironmentVariables(); if (response.Error.HasValue && response.Error.Value) { _logger.LogError(response.Message); return(ApiResult.CreateError(response.Message)); } } if (string.IsNullOrEmpty(diag.Name)) { diag.Name = await _engineEnvironment.GetUniqueName(Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId), diag.DisplayName); } KernelService kernelService = CreateKernelService(diag.Name); //Create the xml with the scala steps to execute to initialize the kernel var hashValue = Helper.GetHashCode(diag.UserName); var sampleDataPath = Helper.SetValueBasedOnSparkType(_engineEnvironment.EngineFlowConfig.SparkType, Path.Combine(_engineEnvironment.OpsSparkSamplePath, $"{diag.Name}-{hashValue}.json"), Helper.ConvertToDbfsFilePath(_engineEnvironment.OpsSparkSamplePath, $"{diag.Name}-{hashValue}.json")); DiagnosticInputhelper(diag.InputSchema, sampleDataPath, diag.NormalizationSnippet, diag.Name); response = await kernelService.RecycleKernelAsync(diag.KernelId, diag.UserName, diag.Name, _engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName), SetupSteps, isReSample, diag.ReferenceDatas, diag.Functions); return(response); } catch (Exception ex) { _logger.LogError(ex, ex.Message); return(ApiResult.CreateError(ex.Message)); } }
/// <summary> /// ExecuteQueryHelper is the function that gets called for executing the user code /// </summary> /// <param name="jObject">jObject as passed in from the frontend</param> /// <returns>Returns the result from the kernel after executing the query</returns> private async Task <ApiResult> ExecuteQueryHelper(JObject jObject) { // validate the Execute Query API parameters i.e. the query object if (jObject == null) { _logger.LogError("Query Object is null"); return(ApiResult.CreateError("Query Object is null")); } var query = jObject.ToObject <InteractiveQueryObject>(); ApiResult response = await _engineEnvironment.GetEnvironmentVariables(); if (response.Error.HasValue && response.Error.Value) { _logger.LogError(response.Message); return(ApiResult.CreateError(response.Message)); } if (string.IsNullOrEmpty(Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId))) { return(ApiResult.CreateError("Subscription can't be null")); } try { KernelService kernelService = CreateKernelService(query.Name); var result = await kernelService.ExecuteQueryAsync(query.Query, query.KernelId); return(result); } catch (Exception ex) { _logger.LogError(ex, ex.Message); return(ApiResult.CreateError(ex.Message)); } }
/// <summary> /// GetSampleInputFromQueryHelper gets the sample from the input table in the user query /// </summary> /// <param name="jObject">jObject </param> /// <returns>Returns the sample result from the input from the user query being executed</returns> private async Task <ApiResult> GetSampleInputFromQueryHelper(JObject jObject) { // validate JObject which is the query object in this case - is not null or empty if (jObject == null) { _logger.LogError("Query Object is null"); return(ApiResult.CreateError("Query Object is null")); } var query = jObject.ToObject <InteractiveQueryObject>(); try { KernelService kernelService = CreateKernelService(query.Name); var result = await kernelService.GetSampleInputFromQueryAsync(query.Query, query.KernelId); return(result); } catch (Exception ex) { _logger.LogError(ex.Message); return(ApiResult.CreateError(ex.Message)); } }
/// <summary> /// This is the API method that gets called from the front end on a regular cadence and will delete all the kernels that were created /// </summary> /// <param name="flowName">flowName</param> /// <returns>Returns the result whether the list of kernels were deleted or not. We don't fail if one of the kernels fails to delete because it just does not exist</returns> public async Task <ApiResult> DeleteAllKernels(string flowName) { var response = await _engineEnvironment.GetEnvironmentVariables().ConfigureAwait(false); if (response.Error.HasValue && response.Error.Value) { _logger.LogError(response.Message); return(ApiResult.CreateError(response.Message)); } KernelService kernelService = CreateKernelService(flowName); _logger.LogInformation("Deleting all Kernels..."); response = await kernelService.GarbageCollectListOfKernels(_engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName), true).ConfigureAwait(false); if (response.Error.HasValue && response.Error.Value) { _logger.LogError(response.Message); return(ApiResult.CreateError(response.Message)); } var result = response.Result.ToString(); _logger.LogInformation("Deleted and cleared all Kernels successfully"); return(ApiResult.CreateSuccess(result)); }
/// <summary> /// This is the API method that gets called from the front end on a regular cadence and will delete all the kernels that are more than 3 hours old /// </summary> /// <param name="flowName">flowName</param> /// <returns>Returns the result whether the list of kernels were deleted or not. We don't fail if one of the kernels fails to delete because it just does not exist</returns> public async Task <ApiResult> DeleteKernels(string flowName) { var response = await _engineEnvironment.GetEnvironmentVariables().ConfigureAwait(false); var subscriptionId = Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId); if (response.Error.HasValue && response.Error.Value) { _logger.LogError(response.Message); return(ApiResult.CreateError(response.Message)); } KernelService kernelService = CreateKernelService(flowName); response = await kernelService.GarbageCollectListOfKernels(_engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName)).ConfigureAwait(false); if (response.Error.HasValue && response.Error.Value) { _logger.LogError(response.Message); return(ApiResult.CreateError(response.Message)); } var result = response.Result.ToString(); return(ApiResult.CreateSuccess(result)); }
/// <summary> /// GetSampleInputFromQueryHelper gets the sample from the input table in the user query /// </summary> /// <param name="jObject">jObject </param> /// <returns>Returns the sample result from the input from the user query being executed</returns> private async Task <ApiResult> GetSampleInputFromQueryHelper(JObject jObject) { // validate JObject which is the query object in this case - is not null or empty if (jObject == null) { _logger.LogError("Query Object is null"); return(ApiResult.CreateError("Query Object is null")); } var query = jObject.ToObject <InteractiveQueryObject>(); try { KernelService kernelService = new KernelService(_engineEnvironment.EngineFlowConfig.JobURLBase, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix, _engineEnvironment.SparkConnInfo, _logger); var result = await kernelService.GetSampleInputFromQueryAsync(query.Query, query.KernelId); return(result); } catch (Exception ex) { _logger.LogError(ex.Message); return(ApiResult.CreateError(ex.Message)); } }