public async Task <ApiResult> RestatAllWithRetries([FromBody] JObject request)
{
try
{
RolesCheck.EnsureWriter(Request, _isLocal);
Ensure.NotNull(request, "request");
var jobNames = request.Value <JArray>("JobNames")?.Value <string[]>() ?? Array.Empty <string>();
var maxRetryTimes = request.GetValue("MaxRetryTimes", StringComparison.OrdinalIgnoreCase)?.Value <int>() ?? 2;
var waitIntervalInSeconds = request.GetValue("WaitIntervalInSeconds", StringComparison.OrdinalIgnoreCase)?.Value <int>() ?? 2;
var maxWaitTimeInSeconds = request.GetValue("MaxWaitTimeInSeconds", StringComparison.OrdinalIgnoreCase)?.Value <int>() ?? 60;
int retryTimes = 0;
IEnumerable <SparkJobFrontEnd> notReadyJobs = Enumerable.Empty <SparkJobFrontEnd>();
do
{
await _jobOperation.RestartAllJobs(jobNames);
var startTime = DateTime.Now;
do
{
SparkJobFrontEnd[] jobOpResult = jobNames != null && jobNames.Any() ?
await _jobOperation.SyncJobStateByNames(jobNames)
: await _jobOperation.SyncAllJobState();
notReadyJobs = jobOpResult.Where(r => r.JobState == JobState.Starting || r.JobState == JobState.Idle || r.JobState == JobState.Error);
jobNames = notReadyJobs.Select(j => j.Name).ToArray();
if (!notReadyJobs.Any())
{
break;
}
System.Threading.Thread.Sleep(waitIntervalInSeconds * 1000);
} while ((DateTime.Now - startTime).TotalSeconds < maxWaitTimeInSeconds);
} while (retryTimes++ < maxRetryTimes && notReadyJobs.Any());
return(ApiResult.CreateSuccess(JToken.FromObject(notReadyJobs)));
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}
/// <summary>
/// This method deleted the list the kernels as passed in
/// </summary>
/// <param name="kernels">The list of kernels to be deleted</param>
/// <param name="subscriptionId">subscriptionId as passed in from the frontend</param>
/// <param name="flowName">flow name</param>
/// <returns>Returns success or error message as the case maybe</returns>
public async Task <ApiResult> DeleteKernelList(List <string> kernels, string flowName)
{
var response = await _engineEnvironment.GetEnvironmentVariables().ConfigureAwait(false);
var subscriptionId = Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
foreach (var k in kernels)
{
await DeleteKernelHelper(subscriptionId, k, flowName).ConfigureAwait(false);
}
return(ApiResult.CreateSuccess("success"));
}
/// <summary>
/// This is the function that gets the sample input from the query as specified by the user for the diagnostic experience
/// </summary>
/// <param name="kernelId">kernelId</param>
/// <param name="code">The code to be executed</param>
/// <returns>ApiResult which contains the actual sample input or the error as the case maybe</returns>
public async Task <ApiResult> GetSampleInputFromQueryAsync(string kernelId, string code)
{
try
{
// Handle CREATE query
if (code.ToLower().StartsWith("create "))
{
return(ApiResult.CreateSuccess("done"));
}
// Attach to kernel
Dictionary <string, string> hs = new Dictionary <string, string>();
string token = Base64Encode($"{_username}:{_password}");
hs.Add("Authorization", $"Basic {token}");
Kernel kernel = new Kernel(kernelId, _baseUrl, null, null, null, hs);
// Prep code to execute
Regex r = new Regex(@"FROM\s*(\w+)");
Match m = r.Match(code);
string g = Guid.NewGuid().ToString().Replace("-", "");
string s = "";
if (m.Success)
{
s = $"val results{g} = sql(\"SELECT * FROM {m.Groups[1].Value} LIMIT {_MaxCount}\"); results{g}.toJSON.take({_MaxCount}).foreach(println)";
}
string result = await Task.Run(() => kernel.ExecuteCode(ReplaceNewLineFeed(s)));
LogErrors(result, ReplaceNewLineFeed(s), "GetSampleInputFromQueryAsync");
if (!string.IsNullOrEmpty(result))
{
return(ConvertToJson(_MaxCount, result));
}
else
{
return(ApiResult.CreateError("{\"Error\":\"No Results\"}"));
}
}
catch (Exception ex)
{
return(ApiResult.CreateError(ex.ToString()));
}
}
public static async Task <ApiResult> DeleteAllBlobsInAContainer(string storageConnectionString, string containerName, string blobDirectory)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
if (storageAccount == null)
{
return(ApiResult.CreateSuccess("Nothing to delete"));
}
CloudBlobContainer container = storageAccount.CreateCloudBlobClient().GetContainerReference(containerName);
bool exists = container.ExistsAsync().Result;
if (!exists)
{
return(ApiResult.CreateSuccess("Nothing to delete"));
}
// Get all the blobs in the container
BlobContinuationToken continuationToken = null;
List <IListBlobItem> blobs = new List <IListBlobItem>();
do
{
var response = container.GetDirectoryReference(blobDirectory).ListBlobsSegmentedAsync(continuationToken).Result;
continuationToken = response.ContinuationToken;
blobs.AddRange(response.Results);
}while (continuationToken != null);
// Delete the blobs
if (blobs == null || blobs.Count <= 0)
{
return(ApiResult.CreateSuccess("Nothing to delete"));
}
foreach (IListBlobItem blob in blobs)
{
await container.GetBlobReference(((CloudBlockBlob)blob).Name).DeleteIfExistsAsync();
}
return(ApiResult.CreateSuccess("Deleted Successfully"));
}
public async Task <ApiResult <Reward> > UpdateRewardServiceAsync(int id, UpdateReward options)
{
if (id < 1)
{
return(new ApiResult <Reward>(
StatusCode.BadRequest, $"not valid {id}"));
}
var result = await context_
.Set <Reward>()
.Where(t => t.ProjectId == id)
.SingleOrDefaultAsync();
if (result == null)
{
return(new ApiResult <Reward>(
StatusCode.NotFound, $"this {result} dont exist"));
}
return(ApiResult <Reward> .CreateSuccess(result));
}
/// <summary>
/// DeleteConfigFromDocumentDB
/// </summary>
/// <param name="db">cosmosDB database</param>
/// <param name="collection">cosmosDB collection</param>
/// <param name="flowId">flowId</param>
/// <returns>ApiResult which contains error or successful result as the case maybe</returns>
public async Task <ApiResult> DeleteConfigFromDocumentDB(string db, string collection, string flowId)
{
if (string.IsNullOrEmpty(db) || string.IsNullOrEmpty(collection))
{
return(ApiResult.CreateError("CosmosDBUtil: db and collection names cannot be null"));
}
try
{
var database = DocumentClient.GetDatabase(db);
var col = database.GetCollection <BsonDocument>(collection);
var filter = Builders <BsonDocument> .Filter.Eq("name", flowId);
var response = await col.DeleteOneAsync(filter);
return(ApiResult.CreateSuccess($"CosmosDBUtil: Deleted the cosmosDB entry for the flow: {flowId}"));
}
catch (Exception ex)
{
return(ApiResult.CreateError("CosmosDBUtil: " + ex.Message));
}
}
/// <summary>
/// CreateKernelAsync - calls into the Rest api for creating the kernel
/// </summary>
/// <returns>ApiResult which contains kernelid</returns>
public async Task <ApiResult> CreateKernelAsync()
{
try
{
// Set body
string body = "{\"name\":\"sparkkernel\"}";
var content = new StringContent(body);
// Call service
var response = await _client.PostAsync($"{_baseUrl}/api/kernels", content);
var responseString = await response.Content.ReadAsStringAsync();
string id = JsonConvert.DeserializeObject <CreateKernelResponse>(responseString).Id;
return(ApiResult.CreateSuccess(id));
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
return(ApiResult.CreateError(ex.ToString()));
}
}
/// <summary>
/// DeleteBlobInStorageContainer deletes the blob in a storage container
/// </summary>
/// <param name="container">CloudBlobContainer container</param>
/// <param name="fileName">fileName</param>
/// <returns></returns>
private async Task <ApiResult> DeleteBlobInStorageContainer(CloudBlobContainer container, string fileName)
{
bool exists = container.ExistsAsync().Result;
if (!exists)
{
return(ApiResult.CreateSuccess("Container does not exist. Nothing to delete"));
}
var blockBlob = container.GetBlockBlobReference(fileName);
bool x = await blockBlob.DeleteIfExistsAsync();
if (x)
{
return(ApiResult.CreateSuccess($"Deleted blob: {fileName}"));
}
else
{
return(ApiResult.CreateError($"Could not delete blob: {fileName}"));
}
}
/// <summary>
/// This method gets called for api/kernel
/// </summary>
/// <param name="jObject">JObject gets passed from the frontend containing the required parameters for this method</param>
/// <returns>Returns the KernelId plus a warning in case the normalization snippet contains columns that are not part of the schema as generated on success. Returns Error in case of error.</returns>
public async Task <ApiResult> CreateAndInitializeKernel(JObject jObject)
{
var diag = jObject.ToObject <InteractiveQueryObject>();
ApiResult response;
if (_engineEnvironment.EngineFlowConfig == null)
{
response = await _engineEnvironment.GetEnvironmentVariables();
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
}
if (string.IsNullOrEmpty(diag.Name))
{
diag.Name = await _engineEnvironment.GetUniqueName(Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId), diag.DisplayName);
}
var hashValue = Helper.GetHashCode(diag.UserName);
string sampleDataPath = Helper.SetValueBasedOnSparkType(_engineEnvironment.EngineFlowConfig.SparkType,
Path.Combine(_engineEnvironment.OpsSparkSamplePath, $"{diag.Name}-{hashValue}.json"),
Helper.ConvertToDbfsFilePath(_engineEnvironment.OpsSparkSamplePath, $"{diag.Name}-{hashValue}.json"));
response = await CreateAndInitializeKernelHelper(diag.InputSchema, diag.UserName, diag.Name, sampleDataPath, diag.NormalizationSnippet, diag.ReferenceDatas, diag.Functions);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
var result = response.Result;
return(ApiResult.CreateSuccess(result));
}
public async Task <ApiResult> RefreshInputDataAndKernel(JObject jObject)
{
var diag = jObject.ToObject <InteractiveQueryObject>();
var response = await _engineEnvironment.GetEnvironmentVariables();
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
//Refresh the sample data
SchemaInferenceManager sim = new SchemaInferenceManager(_logger);
response = await sim.RefreshSample(jObject);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
InteractiveQueryManager iqm = new InteractiveQueryManager(_logger);
response = await iqm.RecycleKernelHelper(diag, true);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
var result = response.Result;
_logger.LogInformation("RefreshInputDataAndKernel successful!");
return(ApiResult.CreateSuccess(result));
}
/// <summary>
/// This is called for deleting the kernel by directly calling into the Rest Api's provided by the jupyter kernel
/// </summary>
/// <param name="kernelId">KernelId</param>
/// <returns>Returns success or error as the case maybe as ApiResult</returns>
public async Task <ApiResult> DeleteKernelAsync(string kernelId)
{
try
{
// Application killed by user.
var response = await _client.DeleteAsync($"{_baseUrl}/api/kernels/{kernelId}");
if (response.IsSuccessStatusCode)
{
return(ApiResult.CreateSuccess("Success"));
}
else
{
var result = await response.Content.ReadAsStringAsync();
return(ApiResult.CreateError(result));
}
}
catch (Exception ex)
{
return(ApiResult.CreateError(ex.ToString()));
}
}
public async Task <ApiResult> GetSchema([FromBody] JObject config)
{
try
{
RolesCheck.EnsureWriter(Request, _isLocal);
Ensure.NotNull(config, "config");
string queries = string.Join("\n", config.GetValue("query").ToString());
var ruleDefinitionRaw = config.Value <JArray>("rules");
var ruleDefinitionList = ruleDefinitionRaw.ToObject <List <FlowGuiRule> >();
var ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(ruleDefinitionList, config.GetValue("name").ToString());
RulesCode codeGen = CodeGen.GenerateCode(queries, ruleDefinitions, config.GetValue("name").ToString());
var result = Analyzer.Analyze(queries, codeGen.Code, config.GetValue("inputSchema").ToString());
await Task.Yield();
return(ApiResult.CreateSuccess(result));
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}
/// <summary>
/// This is the API method that gets called from the front end on a regular cadence and will delete all the kernels that were created
/// </summary>
/// <param name="flowName">flowName</param>
/// <returns>Returns the result whether the list of kernels were deleted or not. We don't fail if one of the kernels fails to delete because it just does not exist</returns>
public async Task <ApiResult> DeleteAllKernels(string flowName)
{
var response = await _engineEnvironment.GetEnvironmentVariables().ConfigureAwait(false);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
KernelService kernelService = CreateKernelService(flowName);
_logger.LogInformation("Deleting all Kernels...");
response = await kernelService.GarbageCollectListOfKernels(_engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName), true).ConfigureAwait(false);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
var result = response.Result.ToString();
_logger.LogInformation("Deleted and cleared all Kernels successfully");
return(ApiResult.CreateSuccess(result));
}
/// <summary>
/// This is the API method that gets called from the front end on a regular cadence and will delete all the kernels that are more than 3 hours old
/// </summary>
/// <param name="flowName">flowName</param>
/// <returns>Returns the result whether the list of kernels were deleted or not. We don't fail if one of the kernels fails to delete because it just does not exist</returns>
public async Task <ApiResult> DeleteKernels(string flowName)
{
var response = await _engineEnvironment.GetEnvironmentVariables().ConfigureAwait(false);
var subscriptionId = Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
KernelService kernelService = CreateKernelService(flowName);
response = await kernelService.GarbageCollectListOfKernels(_engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsDiagnosticPath, _GarbageCollectBlobName)).ConfigureAwait(false);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
var result = response.Result.ToString();
return(ApiResult.CreateSuccess(result));
}
/// <summary>
/// This method gets called for deleting individual kernel as the frontend navigation occurs away from a particular flow for cleanup
/// </summary>
/// <param name="subscriptionId">subscriptionId as passed in from the frontend</param>
/// <param name="kernelId">kernelId as passed in from the frontend</param>
/// <param name="flowName">flowName as passed in from the frontend</param>
/// <returns>Returns success or failure as the case may be</returns>
public async Task <ApiResult> DeleteKernel(string kernelId, string flowName)
{
var response = await _engineEnvironment.GetEnvironmentVariables().ConfigureAwait(false);
var subscriptionId = Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
response = await DeleteKernelHelper(subscriptionId, kernelId, flowName).ConfigureAwait(false);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
var result = response.Result.ToString();
return(ApiResult.CreateSuccess(result));
}
/// This is the function that can be called for deleting all assets created on save of a flow: consumergroup, secrets in Key Vault, cosmosDB products document such that this flow stops showing up in the UI under Flows and blobs
/// Please note if a a new asset is created on Azure as part of saving the flow or any other action taken by user in the UI, this function will need to be updated so that the asset can be deleted on delete of the flow.
/// </summary>
/// <param name="jObject">jObject requires: Subscription; Name; EventhubConnectionString; IsIotHub;EventhubName;userID</param>
/// <returns>Returns result - success or failure as the case maybe</returns>
public async Task <ApiResult> DeleteFlow(JObject jObject)
{
var diag = jObject.ToObject <InteractiveQueryObject>();
ConfigName = diag.Name;
bool errorExists = false;
var response = await _engineEnvironment.GetEnvironmentVariables();
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
return(ApiResult.CreateError(response.Message));
}
///Delete consumer group
_logger.LogInformation($"For FlowId: {ConfigName} Deleting flow specific consumer group.. ");
var inputEventhubConnection = Helper.GetSecretFromKeyvaultIfNeeded(diag.EventhubConnectionString);
_inputEventhubConnectionStringRef = Helper.IsKeyVault(diag.EventhubConnectionString) ? diag.EventhubConnectionString : Helper.GenerateNewSecret(_keySecretList, _engineEnvironment.EngineFlowConfig.SparkKeyVaultName, ConfigName + "-input-eventhubconnectionstring", diag.EventhubConnectionString, false);
diag.EventhubConnectionString = _inputEventhubConnectionStringRef;
if (diag.InputType == Constants.InputType_EventHub)
{
var ehName = Helper.ParseEventHub(inputEventhubConnection);
_eventHubNamespace = Helper.ParseEventHubNamespace(inputEventhubConnection);
_eventHubNameRole = Helper.ParseEventHubPolicyName(inputEventhubConnection);
_eventHubPrimaryKeyListener = Helper.ParseEventHubAccessKey(inputEventhubConnection);
if (string.IsNullOrWhiteSpace(ehName) || string.IsNullOrWhiteSpace(_eventHubNamespace) || string.IsNullOrWhiteSpace(_eventHubNameRole) || string.IsNullOrWhiteSpace(_eventHubPrimaryKeyListener))
{
string error = "The connection string for Event Hub input type must contain Endpoint, SharedAccessKeyName, SharedAccessKey, and EntityPath";
_logger.LogError(error);
errorExists = true;
}
_eventHubNames = new List <string>()
{
ehName
};
}
else
{
_eventHubNames = Helper.ParseEventHubNames(diag.EventhubNames);
_eventHubNamespace = Helper.ParseEventHubNamespace(inputEventhubConnection);
_eventHubNameRole = Helper.ParseEventHubPolicyName(inputEventhubConnection);
_eventHubPrimaryKeyListener = Helper.ParseEventHubAccessKey(inputEventhubConnection);
if (_eventHubNames.Count < 1)
{
string error = "The event hub-compatible name for IoT Hub input type must be defined";
_logger.LogError(error);
errorExists = true;
}
if (string.IsNullOrWhiteSpace(_eventHubNamespace) || string.IsNullOrWhiteSpace(_eventHubNameRole) || string.IsNullOrWhiteSpace(_eventHubPrimaryKeyListener))
{
string error = "The event hub-compatible endpoint for IoT Hub input type must contain Endpoint, SharedAccessKeyName, and SharedAccessKey";
_logger.LogError(error);
errorExists = true;
}
}
// ResourceCreation is one of the environment variables.
// If you don't want to create resource, you can set this to false.
if (_engineEnvironment.ResourceCreation && (diag.InputType == Constants.InputType_EventHub || diag.InputType == Constants.InputType_IoTHub))
{
var inputSubscriptionId = string.IsNullOrEmpty(diag.InputSubscriptionId) ? Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId) : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputSubscriptionId);
var inputResourceGroup = string.IsNullOrEmpty(diag.InputResourceGroup) ? _engineEnvironment.EngineFlowConfig.EventHubResourceGroupName : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputResourceGroup);
foreach (string ehName in _eventHubNames)
{
var result = EventHub.DeleteConsumerGroup(inputSubscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, inputResourceGroup, _engineEnvironment.EngineFlowConfig.EventHubResourceGroupLocation, _eventHubNamespace, ehName, ConsumerGroupName, diag.InputType, _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
if (result.Error.HasValue && result.Error.Value)
{
_logger.LogError(result.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully deleted flow specific consumer group");
}
}
}
///Delete cosmosDB document related to a flow
response = await CosmosDB.DeleteConfigFromDocumentDB(_engineEnvironment.CosmosDBDatabaseName, _engineEnvironment.CosmosDBEndPoint, _engineEnvironment.CosmosDBUserName, _engineEnvironment.CosmosDBPassword, "flows", ConfigName);
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific cosmosDB entry");
}
///Delete configs stored in blobs
// ruleDefinitions
response = await BlobHelper.DeleteBlob(_engineEnvironment.FlowBlobConnectionString, Path.Combine(RuleDefinitionPath, RuleDefinitionFileName));
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific rules definition blob");
}
// outputTemplates
response = await BlobHelper.DeleteBlob(_engineEnvironment.FlowBlobConnectionString, Path.Combine(OutputTemplatePath, OutputTemplateFileName));
if (response.Error.HasValue && response.Error.Value)
{
_logger.LogError(response.Message);
errorExists = true;
}
else
{
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific output template blob");
}
string resourceGroupLocation = _engineEnvironment.EngineFlowConfig.ResourceGroupLocation;
string resourceGroupName = _engineEnvironment.EngineFlowConfig.ResourceGroupName;
string storageAccountName = _engineEnvironment.EngineFlowConfig.StorageAccountName;
string containerPath = Path.Combine(_flowContainerName, _engineEnvironment.EngineFlowConfig.EnvironmentType, ConfigName);
string subscriptionId = Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId);
BlobStorage.DeleteAllConfigsFromBlobStorage(subscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, resourceGroupName, resourceGroupLocation, storageAccountName, _Centralprocessing, Path.Combine(_engineEnvironment.EngineFlowConfig.ContainerPath, ConfigName), _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific blobs under the folder {ConfigName} under container {_Centralprocessing}");
BlobStorage.DeleteAllConfigsFromBlobStorage(subscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, resourceGroupName, resourceGroupLocation, storageAccountName, _Centralprocessing, Path.Combine(_engineEnvironment.EngineFlowConfig.ContainerPath, ConfigName), _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
BlobStorage.DeleteAllConfigsFromBlobStorage(subscriptionId, _engineEnvironment.EngineFlowConfig.ServiceKeyVaultName, resourceGroupName, resourceGroupLocation, storageAccountName, _flowContainerName, Path.Combine(_engineEnvironment.EngineFlowConfig.EnvironmentType, ConfigName), _engineEnvironment.EngineFlowConfig.ConfiggenClientId, _engineEnvironment.EngineFlowConfig.ConfiggenTenantId, _engineEnvironment.EngineFlowConfig.ConfiggenSecretPrefix);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific productconfig: {ProductConfigName} and {JobConfigName} for {ConfigName}.");
/// Delete sample data and the checkpoints folder if it exists for that flow
var hashValue = Helper.GetHashCode(diag.UserName);
await BlobHelper.DeleteBlob(_engineEnvironment.OpsBlobConnectionString, Path.Combine(_engineEnvironment.OpsSamplePath, $"{ConfigName}-{hashValue}.json"));
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific sampledata file: {ConfigName}-{ hashValue}.json");
await BlobHelper.DeleteAllBlobsInAContainer(_engineEnvironment.OpsBlobConnectionString, $"{_engineEnvironment.CheckPointContainerNameHelper(ConfigName)}-checkpoints", _engineEnvironment.EngineFlowConfig.OpsBlobDirectory);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific checkpoints for {ConfigName}.");
_logger.LogInformation("Deleting flow specific secrets..");
///Delete secrets specific to a flow from KeyVault
KeyVault.GetSecretsAndDeleteFromKeyvault(_engineEnvironment.EngineFlowConfig.SparkKeyVaultName, ConfigName);
_logger.LogInformation($"For FlowId: {ConfigName} Successfully Deleted flow specific secrets");
if (!errorExists)
{
return(ApiResult.CreateSuccess("Deleted!"));
}
else
{
return(ApiResult.CreateError("Deleted but with some error. Please check logs for details"));
}
}
/// <summary>
/// Moving the method that sets the various environment variables
/// </summary>
/// <returns>This returns the success or failure as understood by the frontend</returns>
public async Task <ApiResult> GetEnvironmentVariables()
{
CosmosDBDatabaseName = "production";
var cosmosCon = "";
var cosmosDBCollectionName = "";
var response = new ApiResult();
string serviceKeyvaultName = string.Empty;
if (HostUtil.InServiceFabric)
{
response = ServiceFabricUtil.GetServiceKeyVaultName();
if (response.Error.HasValue && response.Error.Value)
{
return(ApiResult.CreateError(response.Message));
}
serviceKeyvaultName = response.Result.ToString();
cosmosCon = KeyVault.GetSecretFromKeyvault(ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigConnectionString").Result.ToString());
CosmosDBDatabaseName = KeyVault.GetSecretFromKeyvault(ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigDatabaseName").Result.ToString());
cosmosDBCollectionName = ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigCollectionName").Result.ToString();
}
else
{
serviceKeyvaultName = _configuration.GetSection(DataXSettingsConstants.ServiceEnvironment).GetSection(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_ServiceKeyVaultName).Value;
cosmosCon = KeyVault.GetSecretFromKeyvault(_configuration.GetSection(DataXSettingsConstants.ServiceEnvironment).GetSection(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_CosmosDBConfigConnectionString).Value);
CosmosDBDatabaseName = KeyVault.GetSecretFromKeyvault(_configuration.GetSection(DataXSettingsConstants.ServiceEnvironment).GetSection(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_CosmosDBConfigDatabaseName).Value);
cosmosDBCollectionName = _configuration.GetSection(DataXSettingsConstants.ServiceEnvironment).GetSection(DataX.Config.ConfigDataModel.Constants.ConfigSettingName_CosmosDBConfigCollectionName).Value;
}
var namePassword = Helper.ParseCosmosDBUserNamePassword(cosmosCon);
if (string.IsNullOrEmpty(cosmosCon) || string.IsNullOrEmpty(namePassword) || namePassword.Split(new char[] { ':' }).Count() != 2)
{
return(ApiResult.CreateError("Can't get UserName or Password from CosmosDB connection string"));
}
CosmosDBEndPoint = Helper.ParseCosmosDBEndPoint(cosmosCon);
CosmosDBUserName = namePassword.Split(new char[] { ':' })[0];
CosmosDBPassword = namePassword.Split(new char[] { ':' })[1];
response = await CosmosDB.DownloadConfigFromDocumentDB(CosmosDBDatabaseName, CosmosDBEndPoint, CosmosDBUserName, CosmosDBPassword, cosmosDBCollectionName).ConfigureAwait(false);
if (response.Error.HasValue && response.Error.Value)
{
return(ApiResult.CreateError(response.Message));
}
var flowConfigObj = response.Result.ToObject <FlowConfigObject>();
if (flowConfigObj != null)
{
EngineFlowConfig = flowConfigObj;
ResourceCreation = flowConfigObj.ResourceCreation.ToLower().Equals("true") ? true : false;
FlowBlobConnectionString = KeyVault.GetSecretFromKeyvault(serviceKeyvaultName, flowConfigObj.ConfiggenSecretPrefix + flowConfigObj.StorageAccountName + "-blobconnectionstring");
OpsBlobConnectionString = KeyVault.GetSecretFromKeyvault(serviceKeyvaultName, flowConfigObj.ConfiggenSecretPrefix + flowConfigObj.OpsStorageAccountName + "-blobconnectionstring");
SparkConnInfo = Helper.ParseConnectionString(Helper.PathResolver(flowConfigObj.SparkConnectionString));
return(ApiResult.CreateSuccess(""));
}
return(ApiResult.CreateError("Failed to get environment variables"));
}
private ApiResult LoadandRunSteps(IKernel kernel, bool isReSample, List <ReferenceDataObject> referenceDatas, List <FunctionObject> functions)
{
try
{
bool normalizationWarning = false;
// Load steps from blob
XmlSerializer ser = new XmlSerializer(typeof(steps));
using (StringReader sreader = new StringReader(_setupStepsXml))
{
using (XmlReader reader = XmlReader.Create(sreader))
{
_steps = (steps)ser.Deserialize(reader);
}
}
string result = "";
if (!isReSample)
{
// Run the steps
for (int i = 0; i < _steps.Items.Count(); i++)
{
Logger.LogInformation($"steps.Items[{i}] ran successfully");
result = kernel.ExecuteCode(_steps.Items[i].Value);
LogErrors(result, _steps.Items[i].Value, $"InitializationStep[{i}]");
_steps.Items[i].Value = NormalizationSnippetHelper(ref i, ref normalizationWarning, result, _steps.Items);
}
}
else
{
// Run the resample steps
for (int i = _steps.Items.Count() - 2; i < _steps.Items.Count(); i++)
{
Logger.LogInformation($"steps.Items[i].Value: _steps.Items[{i}]");
result = kernel.ExecuteCode(_steps.Items[i].Value);
var errors = CheckErrors(result);
if (!string.IsNullOrEmpty(errors))
{
Logger.LogError($"Initialization step: {_steps.Items[i].Value}. Resulting Error: {result}");
}
_steps.Items[i].Value = NormalizationSnippetHelper(ref i, ref normalizationWarning, result, _steps.Items);
}
}
// Now run the steps to load the reference data
LoadReferenceData(kernel, referenceDatas);
// Now load the UDFs and UDAFs
LoadFunctions(kernel, functions);
var error = CheckErrors(result);
if (!string.IsNullOrEmpty(error))
{
return(ApiResult.CreateError("{'Error':'" + error + "'}"));
}
else
{
KernelResult responseObject = new KernelResult
{
Result = kernel.Id
};
if (normalizationWarning)
{
responseObject.Message = "Warning: Normalization query in the Input tab could not be applied, probably because some columns in the query are not part of the schema. Please update the schema or try auto-generating it using longer duration in the Input tab.";
}
else
{
responseObject.Message = string.Empty;
}
return(ApiResult.CreateSuccess(JObject.FromObject(responseObject)));
}
}
catch (Exception ex)
{
Logger.LogError(ex, $"ErrorMessage: {ex.Message} SetupSteps: {_setupStepsXml}");
return(ApiResult.CreateError(ex.ToString()));
}
}
