public async Task <ActionResult <bool> > AddMetricFromDaemonAsync(Guid engineId, [FromBody] YMetricPayload2 payload)
{
var userObjectId = this.User.GetObjectId();
if (string.IsNullOrEmpty(userObjectId))
{
return(new UnauthorizedObjectResult("Daemon id unknown"));
}
if (userObjectId != this.options.ClientObjectId)
{
return(new UnauthorizedObjectResult("This web api should be called only from a daemon application using the correct Client Id / Client Secret"));
}
var engine = await this.engineProvider.GetEngineAsync(engineId).ConfigureAwait(false);
if (engine == null)
{
throw new Exception("Engine does not exists");
}
// Get connection string
var instrumentationKeySecret = await keyVaultsController.GetKeyVaultSecret(engine.Id, engine.AppInsightsName);
string instrumentationKey = instrumentationKeySecret?.Value;
if (string.IsNullOrEmpty(instrumentationKey))
{
throw new Exception("Can't find the App Insights Instrumentation Key in the Key Vault");
}
await PostMetric2Async(engine, payload, instrumentationKey);
return(true);
}
public async Task <ActionResult <YDatabricksCluster> > GetDataBricksWorkspaceClusterAsync(Guid engineId)
{
var engine = await this.engineProvider.GetEngineAsync(engineId).ConfigureAwait(false);
if (engine == null)
{
throw new Exception("Engine does not exists");
}
var resourceGroupName = engine.ResourceGroupName;
var workspaceName = engine.ClusterName;
if (engine == null)
{
throw new Exception("Engine does not exists");
}
// Get Databricks token
var tokenSecret = await keyVaultsController.GetKeyVaultSecret(engine.Id, engine.ClusterName);
string token = tokenSecret?.Value;
if (string.IsNullOrEmpty(token))
{
throw new Exception("Engine has not been created or token is empty");
}
var resourceResponse = await this.resourceClient.GetAsync
(resourceGroupName, "Microsoft.Databricks", "", "workspaces", workspaceName, DataBricksApiVersion);
var workspace = resourceResponse.Value;
// Url for creating a cluster
var dbricksUriBuilder = new UriBuilder($"https://{workspace.Properties["workspaceUrl"]}");
dbricksUriBuilder.Path = "api/2.0/clusters/list";
var dbricksWorkspaceUrl = dbricksUriBuilder.Uri;
// Get all the dbricks cluster already created
var dbricksClustersResponse = await this.client.ProcessRequestAsync <YDatabricksClusters>(dbricksWorkspaceUrl, null, HttpMethod.Get, token);
if (dbricksClustersResponse == null || dbricksClustersResponse.StatusCode != HttpStatusCode.OK || dbricksClustersResponse.Value == null)
{
throw new Exception($"Unable to get the clusters list from Databricks workspace {engine.ClusterName}");
}
var clusterList = dbricksClustersResponse.Value;
var clusterId = string.Empty;
if (clusterList?.Clusters == null || !clusterList.Clusters.Any(c => c.ClusterName == engine.ClusterName))
{
return(new NotFoundResult());
}
var cluster = clusterList.Clusters.First(c => c.ClusterName == engine.ClusterName);
clusterId = cluster.ClusterId;
dbricksUriBuilder.Path = "api/2.0/clusters/get";
dbricksUriBuilder.Query = $"cluster_id={clusterId}";
dbricksWorkspaceUrl = dbricksUriBuilder.Uri;
var clusterStateResponse = await this.client.ProcessRequestAsync <YDatabricksCluster>(dbricksWorkspaceUrl, null, HttpMethod.Get, token);
if (clusterStateResponse == null || clusterStateResponse.StatusCode != HttpStatusCode.OK || clusterStateResponse.Value == null)
{
throw new Exception($"Unable to get cluster {engine.ClusterName} from Databricks workspace {engine.ClusterName}");
}
return(clusterStateResponse.Value);
}
public async Task <ActionResult <JObject> > AddDatabricksLinkService(Guid engineId, string dataSourceName)
{
var engine = await this.engineProvider.GetEngineAsync(engineId).ConfigureAwait(false);
if (engine == null)
{
throw new Exception("Engine does not exists");
}
// Get Databricks token
var tokenSecret = await keyVaultsController.GetKeyVaultSecret(engine.Id, engine.ClusterName);
string token = tokenSecret?.Value;
var resourceGroupName = engine.ResourceGroupName;
var clusterName = engine.ClusterName;
var factoryName = engine.FactoryName;
var pathUri = $"/subscriptions/{options.SubscriptionId}/resourceGroups/{resourceGroupName}" +
$"/providers/Microsoft.DataFactory/factories/{factoryName}" +
$"/linkedservices/{dataSourceName}";
var query = $"api-version={DataFactoryApiVersion}";
var resourceResponse = await this.resourceClient.GetAsync
(resourceGroupName, "Microsoft.Databricks", "", "workspaces", clusterName, DataBricksApiVersion);
var workspace = resourceResponse.Value;
var workspaceUrl = $"https://{workspace.Properties["workspaceUrl"]}";
var dbricksUriBuilder = new UriBuilder(workspaceUrl)
{
Path = "api/2.0/clusters/list"
};
var dbricksWorkspaceUrl = dbricksUriBuilder.Uri;
// Get all the dbricks cluster already created
var dbricksClustersResponse = await this.client.ProcessRequestAsync <YDatabricksClusters>(dbricksWorkspaceUrl, null, HttpMethod.Get, token);
if (dbricksClustersResponse == null || dbricksClustersResponse.StatusCode != HttpStatusCode.OK || dbricksClustersResponse.Value == null)
{
throw new Exception($"Unable to get the clusters list from Databricks workspace {engine.ClusterName}");
}
var clusterList = dbricksClustersResponse.Value;
var clusterId = string.Empty;
if (clusterList?.Clusters == null || !clusterList.Clusters.Any(c => c.ClusterName == engine.ClusterName))
{
return(new NotFoundResult());
}
var cluster = clusterList.Clusters.First(c => c.ClusterName == engine.ClusterName);
clusterId = cluster.ClusterId;
var typeProperties = new JObject
{
{ "properties", new JObject {
{ "type", "AzureDatabricks" },
{ "description", "Databricks Linked Service, created during deployment" },
{ "typeProperties", new JObject {
{ "domain", workspaceUrl },
{ "accessToken", new JObject
{
{ "type", "SecureString" },
{ "value", token }
} },
{ "existingClusterId", clusterId }
} }
} },
};
// Get the response. we may want to create a real class for this result ?
var dbricksTokenResponse = await this.client.ProcessRequestManagementAsync <JObject>(
pathUri, query, typeProperties, HttpMethod.Put).ConfigureAwait(false);
return(dbricksTokenResponse.Value);
}
public async Task <ActionResult <List <YSqlTable> > > GetAzureSqlDatabaseTablesAsync(Guid engineId, string dataSourceName)
{
var engine = await this.engineProvider.GetEngineAsync(engineId).ConfigureAwait(false);
if (engine == null)
{
throw new Exception("Engine does not exists");
}
// Get connection string
var cs = await keyVaultsController.GetKeyVaultSecret(engineId, dataSourceName);
if (cs == null)
{
throw new Exception($"Can't get secret for Data Source {dataSourceName}");
}
var dataSource = await this.dataFactoriesController.GetDataSourceAsync(engineId, dataSourceName);
if (dataSource.Value == null)
{
throw new Exception("Can't get datasource");
}
if (dataSource.Value.DataSourceType != YDataSourceType.AzureSqlDatabase)
{
throw new Exception($"Data Source {dataSourceName} is not a Sql Data Source");
}
var sqlDatabaseSource = new YDataSourceAzureSqlDatabase(dataSource.Value)
{
Password = cs.Value
};
using var sqlConnection = new SqlConnection(sqlDatabaseSource.ConnectionString);
var tableCommandText = @"Select tbl.name as TableName, sch.name as SchemaName
From sys.tables as tbl
Inner join sys.schemas as sch on tbl.schema_id = sch.schema_id";
var sqlCommand = new SqlCommand(tableCommandText, sqlConnection);
var entities = new List <YSqlTable>();
try
{
await sqlConnection.OpenAsync();
using var dr = await sqlCommand.ExecuteReaderAsync();
while (dr.Read())
{
var ysqlTable = new YSqlTable
{
TableName = dr["TableName"].ToString(),
SchemaName = dr["SchemaName"].ToString()
};
entities.Add(ysqlTable);
}
await sqlConnection.CloseAsync();
}
catch (Exception)
{
if (sqlConnection.State != System.Data.ConnectionState.Closed)
{
await sqlConnection.CloseAsync();
}
throw;
}
return(entities);
}