/// <summary>
/// Helper function to delete a kernelList of kernels. If a kernel has already been deleted, then don't return error message and instead continue with the rest of the kernelList and delete the rest of the kernels since the end goal is that the kernelList of kenrels should be deleted.
/// </summary>
/// <param name="kernelList">The kernelList of kernels</param>
/// <returns></returns>
public async Task <ApiResult> GarbageCollectKernelsHelper(List <string> deleteKernelList)
{
try
{
if (deleteKernelList.Count != 0)
{
foreach (string kernelId in deleteKernelList)
{
var response = await DeleteKernelAsync(kernelId);
if (response.Error.HasValue && response.Error.Value)
{
if (!response.Message.Contains($@"Kernel does not exist: {kernelId}") && !response.Message.Contains("404 : Not Found"))
{
return(ApiResult.CreateError(response.Message));
}
}
}
return(ApiResult.CreateSuccess("Successfully deleted the kernelList of kernels"));
}
else
{
return(ApiResult.CreateSuccess("No kernels to delete"));
}
}
catch (Exception ex)
{
return(ApiResult.CreateError(ex.Message));
}
}
public async Task <ApiResult> SaveFlow([FromBody] JObject config)
{
try
{
RolesCheck.EnsureWriter(Request, _isLocal);
Ensure.NotNull(config, "config");
var inputJson = JsonConfig.From(config.ToString());
var result = await _flowOperation.SaveFlowConfig(FlowGuiConfig.From(inputJson));
Ensure.IsSuccessResult(result);
if (result.Properties != null && result.Properties.TryGetValue(FlowOperation.FlowConfigPropertyName, out object flowConfig))
{
return(ApiResult.CreateSuccess(JToken.FromObject(flowConfig)));
}
else
{
return(ApiResult.CreateSuccess(JToken.FromObject(string.Empty)));
}
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}
/// <summary>
/// This deletes the kernelId as specified
/// </summary>
/// <param name="subscriptionId">subscriptionId that is passed in from the frontend</param>
/// <param name="kernelId">kernelId that needs to be deleted</param>
/// <param name="flowName">flowName</param>
/// <returns>Returns success or failure after the delete kernel api is called</returns>
private async Task <ApiResult> DeleteKernelHelper(string subscriptionId, string kernelId, string flowName)
{
// validate KernelId can't be null
if (string.IsNullOrEmpty(kernelId))
{
_logger.LogError("No Kernel found to delete");
return(ApiResult.CreateError("No Kernel found to delete"));
}
try
{
KernelService kernelService = CreateKernelService(flowName);
var result = await kernelService.DeleteKernelAsync(kernelId);
if (!(result.Error.HasValue && result.Error.Value))
{
//Passing in false as the last parameter since this is a delete call and the entry needs to be deleted from the blob
await kernelService.UpdateGarbageCollectKernelBlob(kernelId, "", "", _engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName), false);
}
return(result);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
return(ApiResult.CreateError(ex.Message));
}
}
/// <summary>
/// This method RecycleKernelAsync refreshes the kernel. deletes the old kernel and creates and initializes a new kernel.
/// </summary>
/// <param name="kernelId">kernelId</param>
/// <param name="userId">userId</param>
/// <param name="flowId">flowId</param>
/// <param name="connectionString">SparkConnectionString to Ops is needed for resampling of the data and saving it to the blob</param>
/// <param name="blobUri">blobUri to the blob where the sample data is stored</param>
/// <param name="setupStepsXml">setupStepsXml contains the steps that are needed to run against the kernel</param>
/// <param name="isReSample">boolean that helps identify if it is resample api call or refresh api call</param>
/// <returns>Returns success (KernelId) or error as the case maybe as ApiResult</returns>
public async Task <ApiResult> RecycleKernelAsync(string kernelId, string userId, string flowId, string connectionString, string blobUri, string setupStepsXml, bool isReSample, List <ReferenceDataObject> referenceDatas, List <FunctionObject> functions)
{
try
{
//If it is not a resample call, don't create and delete the kernel
if (!isReSample)
{
//Delete only if the kernelId has been passed in
if (!string.IsNullOrEmpty(kernelId))
{
var deleteResult = await DeleteKernelAsync(kernelId);
}
var response = await GarbageCollectListOfKernels(connectionString, blobUri);
response = await CreateKernelAsync();
if (response.Error.HasValue && response.Error.Value)
{
return(ApiResult.CreateError(response.Message));
}
kernelId = response.Result.ToString();
response = await UpdateGarbageCollectKernelBlob(kernelId, userId, flowId, connectionString, blobUri, true);
}
return(await CreateandInitializeKernelAsync(kernelId, setupStepsXml, isReSample, referenceDatas, functions));
}
catch (Exception ex)
{
return(ApiResult.CreateError(ex.ToString()));
}
}
/// <summary>
/// InitializeKernelAsync - this the helper function that does the actual work of creating and initializing the kernel for the interactive query experience
/// </summary>
/// <param name="kernelId">kernelId</param>
/// <param name="isReSample">isReSample - In case of resampling request from the UI, the actual intitialization is not needed</param>
/// <param name="referenceDatas">referenceDatas: All reference data as specified by the user</param>
/// <param name="functions">All UDFs and UDAFs as specified by the user</param>
/// <returns>ApiResult which contains error or the kernelId (along with/without warning) as the case maybe</returns>
public async Task <ApiResult> InitializeKernelAsync(string kernelId, bool isReSample, List <ReferenceDataObject> referenceDatas, List <FunctionObject> functions)
{
try
{
// Attach to kernel
IKernel kernel = GetKernel(kernelId);
// Load and run the initialization steps
var result = await Task.Run(() => LoadandRunSteps(kernel, isReSample, referenceDatas, functions));
if (result.Error.HasValue && result.Error.Value)
{
return(ApiResult.CreateError(result.Message));
}
else
{
return(ApiResult.CreateSuccess(result.Result));
}
}
catch (Exception ex)
{
Logger.LogError(ex, ex.Message);
return(ApiResult.CreateError(ex.ToString()));
}
}
/// <summary>
/// This is called for deleting the kernel by directly calling into the Rest Api's provided by the jupyter kernel
/// </summary>
/// <param name="kernelId">KernelId</param>
/// <returns>Returns success or error as the case maybe as ApiResult</returns>
public override async Task <ApiResult> DeleteKernelAsync(string kernelId)
{
try
{
if (string.IsNullOrEmpty(kernelId))
{
return(ApiResult.CreateSuccess("Success"));
}
HttpClient client = GetHttpClient();
var response = await client.DeleteAsync($"{_baseUrl}/api/kernels/{kernelId}");
client.Dispose();
if (response.IsSuccessStatusCode)
{
return(ApiResult.CreateSuccess("Success"));
}
else
{
var result = await response.Content.ReadAsStringAsync();
return(ApiResult.CreateError(result));
}
}
catch (Exception ex)
{
return(ApiResult.CreateError(ex.ToString()));
}
}
/// <summary>
/// Returning a json object
/// </summary>
/// <param name="maxCount">Maximum number of rows to return to the UI</param>
/// <param name="result">The Result as returned by the kernel</param>
/// <returns>ApiResult which contains the error message or the json formatted data as returned by kernel</returns>
private ApiResult ConvertToJson(int maxCount, string result)
{
var error = CheckErrors(result);
if (!string.IsNullOrEmpty(error))
{
return(ApiResult.CreateError("{\"Error\": \"" + error + "\"}"));
}
else
{
try
{
var lines = result.Split('\n').ToList();
Regex regex = new Regex(".*: org.apache.spark");
if (lines != null && lines.Count >= 0 && regex.Match(lines[lines.Count - 1]).Success)
{
lines.RemoveAt(lines.Count - 1);
}
var json = JArray.FromObject(lines);
return(ApiResult.CreateSuccess(json.ToString()));
}
catch (Exception)
{
return(ApiResult.CreateError("{\"Error\": \"" + result + "\"}"));
}
}
}
/// <summary>
/// This is a helper function that can be used for updating the blob both when delete and create kernels are called
/// </summary>
/// <param name="kernelId">KernelId is the parameter for the kernel that is being created and deleted</param>
/// <param name="userId">userId</param>
/// <param name="flowId">flowId</param>
/// <param name="connectionString">Ops Connection string that is need to access the blob</param>
/// <param name="blobUri">blobUri of the blob where the kernel data is stored</param>
/// <param name="isCreate">IsCerate is the boolean which tells whether to delete or add an entry in the dictionry in the blob</param>
/// <returns>Returns success or the error message as ApiResult as the case maybe</returns>
public async Task <ApiResult> UpdateGarbageCollectKernelBlob(string kernelId, string userId, string flowId, string connectionString, string blobUri, bool isCreate)
{
try
{
string content = await Task.Run(() => BlobHelper.GetBlobContent(connectionString, blobUri));
Dictionary <string, KernelProperties> kernelPropertiesDict = new Dictionary <string, KernelProperties>();
if (!string.IsNullOrEmpty(content))
{
kernelPropertiesDict = JsonConvert.DeserializeObject <Dictionary <string, KernelProperties> >(content);
}
if (isCreate)
{
kernelPropertiesDict.Add(kernelId, new KernelProperties()
{
UserId = userId, FlowId = flowId, CreateDate = DateTime.Now
});
}
else if (kernelPropertiesDict.Count > 0 && kernelPropertiesDict.ContainsKey(kernelId))
{
kernelPropertiesDict.Remove(kernelId);
}
content = JsonConvert.SerializeObject(kernelPropertiesDict);
await Task.Run(() => BlobHelper.SaveContentToBlob(connectionString, blobUri, content));
return(ApiResult.CreateSuccess("Updated Blob"));
}
catch (Exception ex)
{
return(ApiResult.CreateError(ex.Message));
}
}
public async Task <ApiResult> DownloadConfigFromDocumentDB(string db, string collection)
{
if (string.IsNullOrEmpty(db) || string.IsNullOrEmpty(collection))
{
return(ApiResult.CreateError("CosmosDBUtil: db and collection names cannot be null"));
}
try
{
var database = DocumentClient.GetDatabase(db);
var col = database.GetCollection <BsonDocument>(collection);
var cursor = await col.Find(_ => true).Project(Builders <BsonDocument>
.Projection.Exclude("_id"))
.ToListAsync();
if (cursor != null && cursor.Count() > 0)
{
var doc = cursor.FirstOrDefault();
if (doc != null)
{
return(ApiResult.CreateSuccess(JObject.Parse(doc.ToJson())));
}
}
return(ApiResult.CreateError("CosmosDBUtil: No item with the subscriptionId found"));
}
catch (Exception ex)
{
return(ApiResult.CreateError("CosmosDBUtil: " + ex.Message));
}
}
/// <summary>
/// This is called for deleting the kernel by directly calling into the Rest Api's provided by the jupyter kernel
/// </summary>
/// <param name="kernelId">KernelId</param>
/// <returns>Returns success or error as the case maybe as ApiResult</returns>
public override async Task <ApiResult> DeleteKernelAsync(string kernelId)
{
string responseString = string.Empty;
try
{
InitializeClusterId();
// Set body
string body = "{\"contextId\":\"" + kernelId + "\", \"clusterId\":\"" + _clusterId + "\"}";
var content = new StringContent(body);
// Application killed by user.
HttpClient client = DatabricksUtility.GetHttpClient(_token);
var response = await client.PostAsync($"{_baseUrl}/api/1.2/contexts/destroy", content);
responseString = await response.Content.ReadAsStringAsync();
string id = JsonConvert.DeserializeObject <DeleteDBKernelResponse>(responseString).Id;
client.Dispose();
return(ApiResult.CreateSuccess(id));
}
catch (Exception ex)
{
return(ApiResult.CreateError(responseString + "\n" + ex.ToString()));
}
}
public async Task <ApiResult> UploadConfigToDocumentDB(string db, string collection, JObject item)
{
if (string.IsNullOrEmpty(db) || string.IsNullOrEmpty(collection) || item == null)
{
return(ApiResult.CreateError("CosmosDBUtil: db, collection or item cannot be null"));
}
try
{
var database = DocumentClient.GetDatabase(db);
var col = database.GetCollection <BsonDocument>(collection);
var document = BsonSerializer.Deserialize <BsonDocument>(item.ToString());
var name = item.Value <string>("name");
var filter = Builders <BsonDocument> .Filter.Eq("name", name);
var response = await col.ReplaceOneAsync(filter, document, new UpdateOptions { IsUpsert = true });
return(ApiResult.CreateSuccess(response.UpsertedId != null? response.UpsertedId.ToString() : ""));
}
catch (Exception ex)
{
return(ApiResult.CreateError("CosmosDBUtil: " + ex.Message));
}
}
public async Task <ApiResult> GetCodeGen([FromBody] JObject config)
{
try
{
RolesCheck.EnsureWriter(Request, _isLocal);
Ensure.NotNull(config, "config");
string queries = string.Join("\n", config.GetValue("query").ToString());
var ruleDefinitionRaw = config.Value <JArray>("rules");
var ruleDefinitionList = ruleDefinitionRaw.ToObject <List <FlowGuiRule> >();
var ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(ruleDefinitionList, config.GetValue("name").ToString());
RulesCode codeGen = CodeGen.GenerateCode(queries, ruleDefinitions, config.GetValue("name").ToString());
await Task.Yield();
return(ApiResult.CreateSuccess(codeGen.Code));
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}
/// <summary>
/// This function gets called to get all the data from the configs within a subscription
/// </summary>
/// <param name="subscriptionId">The subscription Id</param>
/// <returns>Returns the configMetadata JObject</returns>
public async Task <ApiResult> GetAllConfigs(string subscriptionId)
{
var response = await GetEnvironmentVariables();
if (response.Error.HasValue && response.Error.Value)
{
return(ApiResult.CreateError(response.Message));
}
try
{
var configs = BlobStorage.LoadAllConfigsFromBlobStorage(subscriptionId, EngineFlowConfig.ServiceKeyVaultName, EngineFlowConfig.ResourceGroupName, EngineFlowConfig.ResourceGroupLocation, EngineFlowConfig.StorageAccountName, _flowContainerName, EngineFlowConfig.EnvironmentType, ProductConfigName, EngineFlowConfig.ConfiggenClientId, EngineFlowConfig.ConfiggenTenantId, EngineFlowConfig.ConfiggenSecretPrefix);
List <ProductConfigMetadata> configMetadata = new List <ProductConfigMetadata>();
foreach (var config in configs)
{
ProductConfigMetadata value = new ProductConfigMetadata();
JObject productConfig = JObject.Parse(config);
value.DisplayName = productConfig.Value <string>("displayName");
value.Owner = productConfig.Value <string>("owner");
value.Name = productConfig.Value <string>("name");
configMetadata.Add(value);
}
return(ApiResult.CreateSuccess(JArray.FromObject(configMetadata)));
}
catch (Exception e)
{
return(ApiResult.CreateError(e.Message));
}
}
/// <summary>
/// InitializeKernelAsync - this the helper function that does the actual work of creating and initializing the kernel for the interactive query experience
/// </summary>
/// <param name="kernelId">kernelId</param>
/// <param name="isReSample">isReSample - In case of resampling request from the UI, the actual intitialization is not needed</param>
/// <param name="referenceDatas">referenceDatas: All reference data as specified by the user</param>
/// <param name="functions">All UDFs and UDAFs as specified by the user</param>
/// <returns>ApiResult which contains error or the kernelId (along with/without warning) as the case maybe</returns>
public async Task <ApiResult> InitializeKernelAsync(string kernelId, bool isReSample, List <ReferenceDataObject> referenceDatas, List <FunctionObject> functions)
{
try
{
// Attach to kernel
Dictionary <string, string> hs = new Dictionary <string, string>();
string token = Base64Encode($"{_username}:{_password}");
hs.Add("Authorization", $"Basic {token}");
Kernel kernel = new Kernel(kernelId, _baseUrl, null, null, null, hs);
// Load and run the initialization steps
var result = await Task.Run(() => LoadandRunSteps(kernel, isReSample, referenceDatas, functions));
if (result.Error.HasValue && result.Error.Value)
{
return(ApiResult.CreateError(result.Message));
}
else
{
return(ApiResult.CreateSuccess(result.Result));
}
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
return(ApiResult.CreateError(ex.ToString()));
}
}
/// <summary>
/// CreateKernelAsync - calls into the Rest api for creating the kernel
/// </summary>
/// <returns>ApiResult which contains kernelid</returns>
public override async Task <ApiResult> CreateKernelAsync()
{
try
{
// Iniitalize cluster id
InitializeClusterId();
// Call service
HttpClient client = DatabricksUtility.GetHttpClient(_token);
var nvc = new List <KeyValuePair <string, string> >();
var url = $"{_baseUrl}/api/1.2/contexts/create?language=scala&clusterId={_clusterId}";
var request = new HttpRequestMessage(HttpMethod.Post, url)
{
Content = new FormUrlEncodedContent(nvc)
};
var response = await client.SendAsync(request);
var responseString = await response.Content.ReadAsStringAsync();
string id = JsonConvert.DeserializeObject <CreateDBKernelResponse>(responseString).Id;
client.Dispose();
return(ApiResult.CreateSuccess(id));
}
catch (Exception ex)
{
Logger.LogError(ex, ex.Message);
return(ApiResult.CreateError(ex.ToString()));
}
}
public async Task <ApiResult> RefreshSample(JObject jObject)
{
var diag = jObject.ToObject <InteractiveQueryObject>();
var eventHubNames = Helper.ParseEventHubNames(diag.EventhubNames);
if (_engineEnvironment.EngineFlowConfig == null)
{
await _engineEnvironment.GetEnvironmentVariables();
}
var connectionString = Helper.GetSecretFromKeyvaultIfNeeded(diag.EventhubConnectionString);
if (diag.InputType == Constants.InputType_EventHub)
{
string ehName = Helper.ParseEventHub(connectionString);
eventHubNames = new List <string>()
{
ehName
};
}
string eventHubNamespace = Helper.ParseEventHubNamespace(connectionString);
// ResourceCreation is one of the environment variables.
// If you don't want to create resource, you can set this to false.
if (_engineEnvironment.ResourceCreation && (diag.InputType == Constants.InputType_EventHub || diag.InputType == Constants.InputType_IoTHub))
{
var inputSubscriptionId = string.IsNullOrEmpty(diag.InputSubscriptionId) ? Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId) : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputSubscriptionId);
var inputResourceGroup = string.IsNullOrEmpty(diag.InputResourceGroup) ? _engineEnvironment.EngineFlowConfig.EventHubResourceGroupName : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputResourceGroup);
// Create consumer group if it doesn't exist
foreach (string ehName in eventHubNames)
{
var result = EventHub.CreateConsumerGroup(inputSubscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, inputResourceGroup, _engineEnvironment.EngineFlowConfig.EventHubResourceGroupLocation, eventHubNamespace, ehName, _engineEnvironment.EngineFlowConfig.ConsumerGroup, diag.InputType, _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
if (result.Error.HasValue && result.Error.Value)
{
_logger.LogError(result.Message);
return(ApiResult.CreateError(result.Message));
}
}
}
if (string.IsNullOrEmpty(diag.Name))
{
diag.Name = await _engineEnvironment.GetUniqueName(Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId), diag.DisplayName);
}
//Fixing a bug where '_' checkpoint container name cannot have '_'
var checkPointContainerName = $"{_engineEnvironment.CheckPointContainerNameHelper(diag.Name)}-checkpoints";
var bootstrapServers = Helper.TryGetBootstrapServers(connectionString);
// Sample events and refresh sample
SchemaGenerator sg = new SchemaGenerator(bootstrapServers, connectionString, eventHubNames, _engineEnvironment.EngineFlowConfig.ConsumerGroup, _engineEnvironment.OpsBlobConnectionString, checkPointContainerName, _engineEnvironment.EngineFlowConfig.OpsBlobDirectory, diag.InputType, diag.InputMode, diag.BatchInputs, _logger);
await sg.RefreshSample(_engineEnvironment.OpsBlobConnectionString, OpsSamplePath, diag.UserName, diag.Name, diag.Seconds);
_logger.LogInformation("Refresh Sample worked!");
return(ApiResult.CreateSuccess("success"));
}
public ApiResult CreateConsumerGroup(string resourceGroupName, string namespaceName, string eventHubName, string consumerGroupName, bool isIotHub = false)
{
if (string.IsNullOrEmpty(namespaceName))
{
throw new Exception("Namespace name is empty!");
}
var consumerGroupParams = new ConsumerGroup();
Console.WriteLine("Creating Consumer Group...");
string errorMessage = "";
//TODO find a way to use only eventhub connection string (without using subscriptionId)
List <string> names = resourceGroupName.Split(',').Select(s => s.Trim()).ToList();
if (names != null && names.Count() > 0)
{
foreach (string name in names)
{
try
{
if (isIotHub)
{
string endpoint = $"https://management.azure.com/subscriptions/{_subscriptionId}/resourceGroups/{name}/providers/Microsoft.Devices/IotHubs/{eventHubName}/eventHubEndpoints/events/ConsumerGroups/{consumerGroupName}?api-version=2018-04-01";
if (CreateIotHubEndpointConsumerGroup(endpoint))
{
return(ApiResult.CreateSuccess("A consumer group is created successfully"));
}
}
else
{
ConsumerGroup found = null;
try
{
found = _eventHubManagementClient.ConsumerGroups.GetAsync(name, namespaceName, eventHubName, consumerGroupName).Result;
}
catch (Exception)
{
}
if (found == null)
{
var r = _eventHubManagementClient.ConsumerGroups.CreateOrUpdateAsync(name, namespaceName, eventHubName, consumerGroupName, consumerGroupParams).Result;
}
return(ApiResult.CreateSuccess("A consumer group is created successfully"));
}
}
catch (Exception ex)
{
errorMessage = ex.InnerException != null ? ex.InnerException.Message : ex.Message;
}
}
}
return(ApiResult.CreateError("Error encountered while creating consumer group: " + consumerGroupName + " Error: " + errorMessage));
}
/// <summary>
/// DeleteConsumerGroup
/// </summary>
/// <param name="resourceGroupName">resourceGroupName</param>
/// <param name="namespaceName">namespaceName</param>
/// <param name="eventHubName">eventHubName</param>
/// <param name="consumerGroupName">consumerGroupName</param>
/// <param name="isIotHub">isIotHub</param>
/// <returns>ApiResult which contains error or successful result as the case maybe</returns>
public ApiResult DeleteConsumerGroup(string resourceGroupName, string namespaceName, string eventHubName, string consumerGroupName, bool isIotHub = false)
{
Console.WriteLine("Deleting Consumer Group...");
//Validate parameters
if (string.IsNullOrEmpty(resourceGroupName) || string.IsNullOrEmpty(namespaceName))
{
return(ApiResult.CreateError("ResourceGroup or Namespace empty while deleting consumer group: " + consumerGroupName));
}
string errorMessage = "";
List <string> names = resourceGroupName.Split(',').Select(s => s.Trim()).ToList();
if (names != null && names.Count() > 0)
{
foreach (string name in names)
{
try
{
if (isIotHub)
{
string endpoint = $"https://management.azure.com/subscriptions/{_subscriptionId}/resourceGroups/{name}/providers/Microsoft.Devices/IotHubs/{eventHubName}/eventHubEndpoints/events/ConsumerGroups/{consumerGroupName}?api-version=2018-04-01";
if (DeleteIotHubEndpointConsumerGroup(endpoint))
{
return(ApiResult.CreateSuccess("A consumer group is deleted successfully"));
}
}
else
{
ConsumerGroup found = null;
try
{
found = _eventHubManagementClient.ConsumerGroups.GetAsync(name, namespaceName, eventHubName, consumerGroupName).Result;
}
catch (Exception)
{
}
if (found != null)
{
var r = _eventHubManagementClient.ConsumerGroups.DeleteAsync(name, namespaceName, eventHubName, consumerGroupName);
}
return(ApiResult.CreateSuccess("A consumer group is deleted successfully"));
}
}
catch (Exception ex)
{
errorMessage = ex.InnerException != null ? ex.InnerException.Message : ex.Message;
}
}
}
return(ApiResult.CreateError("Error encountered while deleting consumer group: " + consumerGroupName + " Error: " + errorMessage));
}
// Helper to get setting value from service fabric settings file
public static ApiResult GetServiceFabricConfigSetting(string settingName)
{
try
{
ConfigurationPackage package = FabricRuntime.GetActivationContext().GetConfigurationPackageObject("Config");
System.Fabric.Description.ConfigurationSection serviceEnvironmentSection = package.Settings.Sections["ServiceEnvironment"];
var settingValue = serviceEnvironmentSection.Parameters[settingName].Value;
return(ApiResult.CreateSuccess(settingValue));
}
catch (Exception ex)
{
return(ApiResult.CreateError(ex.Message));
}
}
/// <summary>
/// CreateAndInitializeKernelHelper is the helper that does the heavy listing for creating and initializing the kernel
/// </summary>
/// <param name="rawSchema">rawSchema as passed in from the frontend</param>
/// <param name="userId">userId as passed in from the frontend</param>
/// <param name="flowId">flowId as passed in from the frontend</param>
/// <param name="sampleDataPath">sampleDataPath where the sample data is stored</param>
/// <param name="normalizationSnippet">normalizationSnippet as passed in from the frontend</param>
/// <param name="referenceDatas">referenceDatas as passed in from the frontend</param>
/// <param name="functions">functions as passed in from the frontend</param>
/// <returns></returns>
private async Task <ApiResult> CreateAndInitializeKernelHelper(string rawSchema, string userId, string flowId, string sampleDataPath, string normalizationSnippet, List <ReferenceDataObject> referenceDatas, List <FunctionObject> functions)
{
try
{
//Create the xml with the scala steps to execute to initialize the kernel
DiagnosticInputhelper(rawSchema, sampleDataPath, normalizationSnippet, flowId);
KernelService kernelService = CreateKernelService(flowId);
var response = await kernelService.GarbageCollectListOfKernels(_engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName));
response = await kernelService.CreateKernelAsync();
if (response.Error.HasValue && response.Error.Value)
{
return(ApiResult.CreateError(response.Message));
}
string kernelId = response.Result.ToString();
response = await kernelService.UpdateGarbageCollectKernelBlob(kernelId, userId, flowId, _engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName), true);
if (response.Error.HasValue && response.Error.Value)
{
await kernelService.DeleteKernelAsync(kernelId);
return(ApiResult.CreateError(response.Message));
}
if (_engineEnvironment.EngineFlowConfig.SparkType == Config.ConfigDataModel.Constants.SparkTypeDataBricks)
{
kernelService.MountStorage(_engineEnvironment.EngineFlowConfig.OpsStorageAccountName, _engineEnvironment.EngineFlowConfig.SparkKeyVaultName, sampleDataPath, kernelId);
}
response = await kernelService.CreateandInitializeKernelAsync(kernelId, SetupSteps, false, referenceDatas, functions);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
else
{
return(ApiResult.CreateSuccess(response.Result));
}
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
return(ApiResult.CreateError(ex.Message));
}
}
/// <summary>
/// This is the function that gets the kernelList of kernels from the blob that are older than 3 hours and deletes them
/// </summary>
/// <param name="connectionString">Pass in the connection string for the Ops blob storage where there is a kernelList kernels mantained as they are created</param>
/// <param name="blobUri">blobUri: This is the Uri to the diagnostic blob where the kernelList all created kernels is maintained</param>
/// <returns>Returns the result of delete call. If this fails because a kernel does not exist, it does not fail but continues to delete the remaining kernels</returns>
public async Task <ApiResult> GarbageCollectListOfKernels(string connectionString, string blobUri, bool isDeleteAll = false)
{
string content = await Task.Run(() => BlobHelper.GetBlobContent(connectionString, blobUri));
List <string> kernelList = new List <string>();
if (!string.IsNullOrEmpty(content))
{
Dictionary <string, KernelProperties> kernelPropertiesDict = JsonConvert.DeserializeObject <Dictionary <string, KernelProperties> >(content);
foreach (KeyValuePair <string, KernelProperties> kvp in kernelPropertiesDict)
{
if (isDeleteAll)
{
kernelList.Add(kvp.Key);
}
else
{
//A list gets generated for all kernels that are more than 3 hours old. Front end needs to call this api on a regular cadence
if (DateTime.Now.AddHours(-3) > kvp.Value.CreateDate)
{
kernelList.Add(kvp.Key);
}
}
}
var response = await GarbageCollectKernelsHelper(kernelList);
if (response.Error.HasValue && response.Error.Value)
{
return(response);
}
foreach (string kerId in kernelList)
{
kernelPropertiesDict.Remove(kerId);
}
if (response.Error.HasValue && response.Error.Value)
{
return(ApiResult.CreateError(response.Message));
}
content = JsonConvert.SerializeObject(kernelPropertiesDict);
await Task.Run(() => BlobHelper.SaveContentToBlob(connectionString, blobUri, content));
return(response);
}
else
{
return(ApiResult.CreateSuccess("No Kernels to delete"));
}
}
[Route("inputdata/refreshsample")] // schema inf
public async Task <ApiResult> RefreshSample([FromBody] JObject jObject)
{
try
{
RolesCheck.EnsureWriter(Request);
SchemaInferenceManager sim = new SchemaInferenceManager(_logger);
return(await sim.RefreshSample(jObject));
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}
[Route("inputdata/refreshsampleandkernel")] // diag
public async Task <ApiResult> RefreshInputDataAndKernel([FromBody] JObject jObject)
{
try
{
RolesCheck.EnsureWriter(Request);
LiveDataManager ldm = new LiveDataManager(_logger, _configuration);
return(await ldm.RefreshInputDataAndKernel(jObject));
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}
[Route("kernel/executequery")] // diag
public async Task <ApiResult> ExecuteCode([FromBody] JObject jObject)
{
try
{
RolesCheck.EnsureWriter(Request);
InteractiveQueryManager iqm = new InteractiveQueryManager(_logger);
return(await iqm.ExecuteQuery(jObject));
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}
/// <summary>
/// ExecuteQuery method is called into when the api/kernel/execute api is called from the frontend
/// </summary>
/// <param name="queryObject">The query object is passed in</param>
/// <returns>Returns the result from actually executing the query.</returns>
public async Task <ApiResult> ExecuteQuery(JObject queryObject)
{
var response = await ExecuteQueryHelper(queryObject);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
var result = response.Result.ToString();
return(ApiResult.CreateSuccess(result));
}
/// <summary>
/// Moving the method that sets the various environment variables
/// </summary>
/// <returns>This returns the success or failure as understood by the frontend</returns>
public async Task <ApiResult> GetEnvironmentVariables()
{
CosmosDBDatabaseName = "production";
var response = ServiceFabricUtil.GetServiceKeyVaultName();
if (response.Error.HasValue && response.Error.Value)
{
return(ApiResult.CreateError(response.Message));
}
string serviceKeyvaultName = response.Result.ToString();
var cosmosCon = KeyVault.GetSecretFromKeyvault(ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigConnectionString").Result.ToString());
CosmosDBDatabaseName = KeyVault.GetSecretFromKeyvault(ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigDatabaseName").Result.ToString());
var namePassword = Helper.ParseCosmosDBUserNamePassword(cosmosCon);
if (string.IsNullOrEmpty(cosmosCon) || string.IsNullOrEmpty(namePassword) || namePassword.Split(new char[] { ':' }).Count() != 2)
{
return(ApiResult.CreateError("Can't get UserName or Password from CosmosDB connection string"));
}
CosmosDBEndPoint = Helper.ParseCosmosDBEndPoint(cosmosCon);
CosmosDBUserName = namePassword.Split(new char[] { ':' })[0];
CosmosDBPassword = namePassword.Split(new char[] { ':' })[1];
response = await CosmosDB.DownloadConfigFromDocumentDB(CosmosDBDatabaseName, CosmosDBEndPoint, CosmosDBUserName, CosmosDBPassword, ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigCollectionName").Result.ToString());
if (response.Error.HasValue && response.Error.Value)
{
return(ApiResult.CreateError(response.Message));
}
var flowConfigObj = response.Result.ToObject <FlowConfigObject>();
if (flowConfigObj != null)
{
EngineFlowConfig = flowConfigObj;
ResourceCreation = flowConfigObj.ResourceCreation.ToLower().Equals("true") ? true : false;
FlowBlobConnectionString = KeyVault.GetSecretFromKeyvault(serviceKeyvaultName, flowConfigObj.ConfiggenSecretPrefix + flowConfigObj.StorageAccountName + "-blobconnectionstring");
OpsBlobConnectionString = KeyVault.GetSecretFromKeyvault(serviceKeyvaultName, flowConfigObj.ConfiggenSecretPrefix + flowConfigObj.OpsStorageAccountName + "-blobconnectionstring");
SparkConnInfo = Helper.ParseConnectionString(Helper.PathResolver(flowConfigObj.SparkConnectionString));
return(ApiResult.CreateSuccess(""));
}
return(ApiResult.CreateError("Failed to get environment variables"));
}
private static async Task <ApiResult> SaveSecretToKeyValutAsync(string keyvaultName, string name, string value)
{
KeyVaultManager keyVaultManager = new KeyVaultManager();
try
{
var secret = await keyVaultManager.SaveSecretStringAsync(keyvaultName, name, value);
return(ApiResult.CreateSuccess(JToken.FromObject(secret)));
}
catch (Exception ex)
{
return(ApiResult.CreateError(ex.Message));
}
}
[Route("kernel/refresh")] // diag
public async Task <ApiResult> RefreshKernel([FromBody] JObject jObject)
{
try
{
RolesCheck.EnsureWriter(Request);
InteractiveQueryManager iqm = new InteractiveQueryManager(_logger, _configuration);
return(await iqm.RecycleKernel(jObject));
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}
[Route("flow/schedulebatch")] // schedule batch jobs
public async Task <ApiResult> ScheduleBatch()
{
try
{
RolesCheck.EnsureWriter(Request, _isLocal);
var result = await _flowOperation.ScheduleBatch(this._runtimeConfigGenerator).ConfigureAwait(false);
return(ApiResult.CreateSuccess(JToken.FromObject(result)));
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}
public async Task <ApiResult> DeleteAllKernels()
{
try
{
RolesCheck.EnsureWriter(Request);
InteractiveQueryManager iqm = new InteractiveQueryManager(_logger);
return(await iqm.DeleteAllKernels().ConfigureAwait(false));
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}