public async Task <ActionResult <JArray> > GetStorageDfsFilesAsync(Guid engineId, string dataSourceName)
{
var engine = await this.engineProvider.GetEngineAsync(engineId).ConfigureAwait(false);
if (engine == null)
{
throw new Exception("Engine does not exists");
}
var dataSourceResponse = await this.dataFactoriesController.GetDataSourceAsync(engineId, dataSourceName);
if (dataSourceResponse.Value == null)
{
throw new Exception("DataSource does not exists");
}
var dataSource = dataSourceResponse.Value;
if (dataSource.DataSourceType != YDataSourceType.AzureBlobFS && dataSource.DataSourceType != YDataSourceType.AzureBlobStorage)
{
throw new Exception("DataSource is not a Azure Blob or Azure Data Lake Gen2 Data Source.");
}
// Get typed instance to get the correct call to GetSensitiveString()
var typeDataSource = YDataSourceFactory.GetTypedDatSource(dataSource) as YDataSourceAzureBlob;
var accountKey = await this.keyVaultsController.GetKeyVaultSecret(engineId, dataSource.Name);
if (accountKey == null || accountKey.Value == null)
{
throw new Exception("DataSource Account key is not present in th keyvault");
}
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(typeDataSource.StorageAccountName, accountKey.Value);
var root = new JArray();
if (typeDataSource.DataSourceType == YDataSourceType.AzureBlobStorage)
{
// be careful, to get account detail, we are targeting ".bob." and not ".dfs"
string dfsUri = "https://" + typeDataSource.StorageAccountName + ".blob.core.windows.net";
var blobServiceClient = new BlobServiceClient(new Uri(dfsUri), sharedKeyCredential);
AsyncPageable <BlobContainerItem> allContainers = blobServiceClient.GetBlobContainersAsync();
await foreach (var containerItem in allContainers)
{
var blobContainerClient = blobServiceClient.GetBlobContainerClient(containerItem.Name);
ListBlobsHierarchicalListing(blobContainerClient, default, default, ref root);
public async Task <ActionResult <YDataSource> > AddDataSourceAsync(Guid engineId,
string dataSourceName, [FromBody] YDataSourceUnknown dataSource)
{
var engine = await this.engineProvider.GetEngineAsync(engineId).ConfigureAwait(false);
if (engine == null)
{
throw new Exception("Engine does not exists");
}
var regex = new Regex(@"^[a-zA-Z0-9--]{3,24}$");
if (!regex.IsMatch(dataSourceName))
{
throw new Exception($"DataSource name {dataSourceName} is incorrect");
}
var resourceGroupName = engine.ResourceGroupName;
var factoryName = engine.FactoryName;
var pathUri = $"/subscriptions/{options.SubscriptionId}" +
$"/resourceGroups/{resourceGroupName}/providers/Microsoft.DataFactory" +
$"/factories/{factoryName}/linkedservices/{dataSourceName}";
var query = $"api-version={DataFactoryApiVersion}";
var typedDataSource = YDataSourceFactory.GetTypedDatSource(dataSource);
// get sensitive string if any
string sensitiveString = typedDataSource.GetSensitiveString();
if (!string.IsNullOrEmpty(sensitiveString))
{
// Save the connection string to KeyVault
await keyVaultsController.SetKeyVaultSecret(engineId, dataSourceName,
new YKeyVaultSecretPayload { Key = dataSourceName, Value = sensitiveString });
}
// Get the response. we may want to create a real class for this result ?
var response = await this.client.ProcessRequestManagementAsync <YDataSourceUnknown>(
pathUri, query, typedDataSource, HttpMethod.Put).ConfigureAwait(false);
return(response.Value);
}
