public void CanSetNonGenericTypePropertiesWithoutTypeName()
{
var storageLinkedService = new AzureStorageLinkedService();
Assert.DoesNotThrow(
() =>
{
var ls = new LinkedServiceProperties(storageLinkedService);
ls.TypeProperties = storageLinkedService;
});
}
public override LinkedService ToSdkObject()
{
var linkedService = new AzureStorageLinkedService();
linkedService.ConnectionString = this.ConnectionString;
linkedService.AccountKey = this.AccountKey;
linkedService.SasUri = this.SasUri;
linkedService.SasToken = this.SasToken;
linkedService.EncryptedCredential = this.EncryptedCredential;
SetProperties(linkedService);
return(linkedService);
}
/// <summary>
/// Delete azure blob file or entire folder
/// </summary>
/// <param name="dataSetsToDelete"></param>
///
public void DeleteBlobFileFolder(List <string> dataSetsToDelete)
{
foreach (string strInputToDelete in dataSetsToDelete)
{
Dataset inputDataset = datasets.First(ds => ds.Name.Equals(strInputToDelete));
AzureBlobDataset blobDataset = inputDataset.Properties.TypeProperties as AzureBlobDataset;
logger.Write("\nBlob folder: " + blobDataset.FolderPath);
logger.Write("\nBlob file: " + blobDataset.FileName);
// linked service for input and output is the same.
AzureStorageLinkedService linkedService = linkedServices.First(ls =>
ls.Name == inputDataset.Properties.LinkedServiceName).Properties.TypeProperties as AzureStorageLinkedService;
// create storage client for input. Pass the connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(linkedService.ConnectionString);
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
// find blob to delete and delete if exists.
Uri blobUri = new Uri(storageAccount.BlobEndpoint, blobDataset.FolderPath + blobDataset.FileName);
CloudBlockBlob blob = new CloudBlockBlob(blobUri, storageAccount.Credentials);
logger.Write("Blob Uri: {0}", blobUri.AbsoluteUri);
logger.Write("Blob exists: {0}", blob.Exists());
blob.DeleteIfExists();
logger.Write("Deleted blob: {0}", blobUri.AbsoluteUri);
// Ensure the container is exist.
if (blobDataset.FolderPath.IndexOf("/") > 0)
{
string containerName = blobDataset.FolderPath.Substring(0, blobDataset.FolderPath.IndexOf("/"));
logger.Write("Container Name {0}", containerName);
string directoryName = blobDataset.FolderPath.Substring(blobDataset.FolderPath.IndexOf("/") + 1);
logger.Write("Directory Name {0}", directoryName);
var blobContainer = client.GetContainerReference(containerName);
blobContainer.CreateIfNotExists();
CloudBlobDirectory cbd = blobContainer.GetDirectoryReference(directoryName);
foreach (IListBlobItem item in blobContainer.ListBlobs(directoryName, true))
{
logger.Write("Blob Uri: {0} ", item.Uri.AbsoluteUri);
if (item.GetType() == typeof(CloudBlockBlob) || item.GetType().BaseType == typeof(CloudBlockBlob))
{
CloudBlockBlob subBlob = new CloudBlockBlob(item.Uri, storageAccount.Credentials);
logger.Write("Blob exists: {0}", subBlob.Exists());
subBlob.DeleteIfExists();
logger.Write("Deleted blob {0}", item.Uri.AbsoluteUri);
}
}
}
}
}
// Get Linked Service Connection String
private static string GetConnectionString(LinkedService asset)
{
if (asset == null)
{
return(null);
}
AzureStorageLinkedService storageAsset = asset.Properties as AzureStorageLinkedService;
if (storageAsset == null)
{
return(null);
}
return(storageAsset.ConnectionString);
}
private static void WriteDataToBlob(
Dataset outputDataset,
AzureStorageLinkedService outputLinkedService,
IActivityLogger logger,
string csvData
)
{
var folderPath = GetFolderPath(outputDataset);
// create a storage object for the output blob.
var outputConnectionString = outputLinkedService.ConnectionString;
var outputStorageAccount = CloudStorageAccount.Parse(outputConnectionString);
var outputBlobUri = new Uri(outputStorageAccount.BlobEndpoint, folderPath + "/" + GetFileName(outputDataset));
var outputBlob = new CloudBlockBlob(outputBlobUri, outputStorageAccount.Credentials);
outputBlob.UploadText(csvData);
}
public async Task <ActionResult <LinkedServiceResource> > AddDataSource2Async(Guid engineId,
string dataSourceName, [FromBody] LinkedService linkedService)
{
var engine = await this.engineProvider.GetEngineAsync(engineId).ConfigureAwait(false);
if (engine == null)
{
throw new Exception("Engine does not exists");
}
var accessToken = await this.authProvider.GetAccessTokenForAppManagementAsync().ConfigureAwait(false);
ServiceClientCredentials tokenCredentials = new TokenCredentials(accessToken);
DataFactoryManagementClient client = new DataFactoryManagementClient(tokenCredentials)
{
SubscriptionId = this.options.SubscriptionId
};
// Create the linked service resource
LinkedServiceResource linkedServiceResource = new LinkedServiceResource(linkedService);
string sensitiveString = linkedServiceResource.Properties switch
{
AzureStorageLinkedService azureStorageLinkedService => azureStorageLinkedService.ConnectionString.ToString(),
_ => null
};
if (!string.IsNullOrEmpty(sensitiveString))
{
// Save the connection string to KeyVault
await keyVaultsController.SetKeyVaultSecret(engineId, dataSourceName,
new YKeyVaultSecretPayload { Key = dataSourceName, Value = sensitiveString });
}
var newLinkedServiceResourceCreated = await client.LinkedServices.CreateOrUpdateAsync(engine.ResourceGroupName, engine.FactoryName, dataSourceName, linkedServiceResource);
return(newLinkedServiceResourceCreated);
}
public IDictionary <string, string> Execute(IEnumerable <LinkedService> linkedServices,
IEnumerable <Dataset> datasets,
Activity activity,
IActivityLogger logger)
{
try
{
logger.Write("Custom Activity Started.");
DotNetActivity dotNetActivity = (DotNetActivity)activity.TypeProperties;
string inputToDelete = dotNetActivity.ExtendedProperties["InputToDelete"];
logger.Write("\nInput to delete is " + inputToDelete);
logger.Write("\nAll Dataset(s) Below ");
foreach (Dataset ds in datasets)
{
logger.Write("\nDataset: " + ds.Name);
}
foreach (string name in activity.Inputs.Select(i => i.Name))
{
logger.Write("\nInput Dataset: " + name);
}
foreach (string name in activity.Outputs.Select(i => i.Name))
{
logger.Write("\nOutput Dataset: " + name);
}
List <string> dataSetsToDelete = inputToDelete.Split(',').ToList();
foreach (string strInputToDelete in dataSetsToDelete)
{
Dataset inputDataset = datasets.First(ds => ds.Name.Equals(strInputToDelete));
AzureBlobDataset blobDataset = inputDataset.Properties.TypeProperties as AzureBlobDataset;
logger.Write("\nBlob folder: " + blobDataset.FolderPath);
logger.Write("\nBlob file: " + blobDataset.FileName);
// linked service for input and output is the same.
AzureStorageLinkedService linkedService = linkedServices.First(ls =>
ls.Name == inputDataset.Properties.LinkedServiceName).Properties.TypeProperties as AzureStorageLinkedService;
// create storage client for input. Pass the connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(linkedService.ConnectionString);
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
// find blob to delete and delete if exists.
Uri blobUri = new Uri(storageAccount.BlobEndpoint, blobDataset.FolderPath + blobDataset.FileName);
CloudBlockBlob blob = new CloudBlockBlob(blobUri, storageAccount.Credentials);
logger.Write("Blob Uri: {0}", blobUri.AbsoluteUri);
logger.Write("Blob exists: {0}", blob.Exists());
blob.DeleteIfExists();
logger.Write("Deleted blob: {0}", blobUri.AbsoluteUri);
// Ensure the container is exist.
if (blobDataset.FolderPath.IndexOf("/") > 0)
{
string containerName = blobDataset.FolderPath.Substring(0, blobDataset.FolderPath.IndexOf("/"));
logger.Write("Container Name {0}", containerName);
string directoryName = blobDataset.FolderPath.Substring(blobDataset.FolderPath.IndexOf("/") + 1);
logger.Write("Directory Name {0}", directoryName);
var blobContainer = client.GetContainerReference(containerName);
blobContainer.CreateIfNotExists();
CloudBlobDirectory cbd = blobContainer.GetDirectoryReference(directoryName);
foreach (IListBlobItem item in blobContainer.ListBlobs(directoryName, true))
{
logger.Write("Blob Uri: {0} ", item.Uri.AbsoluteUri);
if (item.GetType() == typeof(CloudBlockBlob) || item.GetType().BaseType == typeof(CloudBlockBlob))
{
CloudBlockBlob subBlob = new CloudBlockBlob(item.Uri, storageAccount.Credentials);
logger.Write("Blob exists: {0}", subBlob.Exists());
subBlob.DeleteIfExists();
logger.Write("Deleted blob {0}", item.Uri.AbsoluteUri);
}
}
}
}
logger.Write("Custom Activity Ended Successfully.");
}
catch (Exception e)
{
logger.Write("Custom Activity Failed with error.");
logger.Write("Caught exception: ");
logger.Write(e.Message);
throw new Exception(e.Message);
}
// The dictionary can be used to chain custom activities together in the future.
// This feature is not implemented yet, so just return an empty dictionary.
return(new Dictionary <string, string>());
}
public static DotNetActivityContext DeserializeActivity(string pipelineFileName, string activityName, string configFile = null, string adfFilesPath = @"..\..\..\McioppDataFactory")
{
// Get Key Vault settings if secure publish is being used on the local machine
AdfFileHelper adfFileHelper = null;
string settingsFile = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "SecurePublishSettings.json");
if (File.Exists(settingsFile))
{
AppSettings settings = JsonConvert.DeserializeObject <AppSettings>(File.ReadAllText(settingsFile));
X509Certificate2 cert = KeyVaultResolver.FindCertificateByThumbprint(settings.KeyVaultCertThumbprint);
string suffix = settings.EnvironmentSettings.First().KeyVaultDnsSuffix;
suffix = string.IsNullOrEmpty(suffix) ? "vault.azure.net:443" : suffix;
KeyVaultResolver keyVaultResolver = new KeyVaultResolver(settings.EnvironmentSettings.First().KeyVaultName, suffix, settings.KeyVaultCertClientId, cert);
adfFileHelper = new AdfFileHelper(keyVaultResolver, new Logger());
}
adfFilesPath = Path.GetFullPath(adfFilesPath);
var deploymentDict = new Dictionary <string, Dictionary <string, string> >();
if (!string.IsNullOrEmpty(configFile))
{
// Get deployment config
string deploymentConfigPath = Path.Combine(adfFilesPath, configFile);
var deploymentConfigJson = File.ReadAllText(deploymentConfigPath);
var deploymentJObj = JObject.Parse(deploymentConfigJson);
deploymentDict = deploymentJObj.Properties()
.ToDictionary(x => x.Name,
y => y.Value.ToDictionary(z => z["name"].ToString(), z => z["value"].ToString()));
}
DotNetActivityContext context = new DotNetActivityContext
{
LinkedServices = new List <LinkedService>(),
Datasets = new List <Dataset>(),
Activity = new Activity(),
Logger = new ActivityLogger()
};
string pipelinePath = Path.Combine(adfFilesPath, pipelineFileName);
string pipelineJson = File.ReadAllText(pipelinePath);
string pipelineName = Path.GetFileNameWithoutExtension(pipelineFileName);
// Update with values from delpoyment config if exists
if (deploymentDict.Count > 0 && deploymentDict.ContainsKey(pipelineName))
{
JObject pipelineJObject = JObject.Parse(pipelineJson);
foreach (KeyValuePair <string, string> pair in deploymentDict[pipelineName])
{
JToken token = pipelineJObject.SelectToken(pair.Key);
token.Replace(pair.Value);
}
pipelineJson = pipelineJObject.ToString();
}
// Search for Key Vault references in the pipeline and replace with their Key Vault equivalents if found
if (adfFileHelper != null)
{
pipelineJson = adfFileHelper.ResolveKeyVault(pipelineJson).Result;
}
var dummyPipeline = JsonConvert.DeserializeObject <Models.Pipeline>(pipelineJson);
Models.Activity dummyActivity;
try
{
dummyActivity = dummyPipeline.Properties.Activities.Single(x => x.Name == activityName);
}
catch (InvalidOperationException)
{
throw new Exception($"Activity {activityName} not found in {pipelinePath}.");
}
context.Activity.Name = dummyActivity.Name;
context.Activity.TypeProperties = new DotNetActivity();
DotNetActivity dotNetActivity = (DotNetActivity)context.Activity.TypeProperties;
dotNetActivity.ExtendedProperties = dummyActivity.DotNetActivityTypeProperties.ExtendedProperties;
// get the input and output tables
var dummyDatasets = new HashSet <Models.ActivityData>();
dummyDatasets.UnionWith(dummyActivity.Inputs);
dummyDatasets.UnionWith(dummyActivity.Outputs);
var dummyServices = new HashSet <Models.LinkedService>();
// init the data tables
foreach (var dummyDataset in dummyDatasets)
{
// parse the table json source
var dataPath = Path.Combine(adfFilesPath, dummyDataset.Name + ".json");
var dataJson = File.ReadAllText(dataPath);
var dummyTable = JsonConvert.DeserializeObject <Models.Table>(dataJson);
{
// initialize dataset properties
DatasetTypeProperties datasetProperties;
switch (dummyTable.Properties.Type)
{
case "AzureBlob":
// init the azure model
var blobDataset = new AzureBlobDataset
{
FolderPath = dummyTable.Properties.TypeProperties.FolderPath,
FileName = dummyTable.Properties.TypeProperties.FileName
};
datasetProperties = blobDataset;
break;
case "AzureTable":
case "AzureSqlTable":
var tableDataset = new AzureTableDataset
{
TableName = dummyTable.Properties.TypeProperties.TableName
};
datasetProperties = tableDataset;
break;
case "SqlServerTable":
var sqlTableDataset = new SqlServerTableDataset(dummyTable.Properties.TypeProperties.TableName);
datasetProperties = sqlTableDataset;
break;
default:
throw new Exception($"Unexpected Dataset.Type {dummyTable.Properties.Type}");
}
// initialize dataset
var dataDataset = new Dataset(
dummyDataset.Name,
new DatasetProperties(
datasetProperties,
new Availability(),
string.Empty
)
);
dataDataset.Properties.LinkedServiceName = dummyTable.Properties.LinkedServiceName;
context.Datasets.Add(dataDataset);
}
// register the inputs and outputs in the activity
if (dummyDataset is Models.ActivityInput)
{
context.Activity.Inputs.Add(new ActivityInput(dummyDataset.Name));
}
if (dummyDataset is Models.ActivityOutput)
{
context.Activity.Outputs.Add(new ActivityOutput(dummyDataset.Name));
}
// parse the linked service json source for later use
string linkedServiceName = dummyTable.Properties.LinkedServiceName;
var servicePath = Path.Combine(adfFilesPath, linkedServiceName + ".json");
string serviceJson = File.ReadAllText(servicePath);
string linkedServiceType = string.Empty;
// Update with values from delpoyment config if exists
if (deploymentDict.Count > 0 && deploymentDict.ContainsKey(linkedServiceName))
{
JObject serviceJObject = JObject.Parse(serviceJson);
linkedServiceType = serviceJObject["properties"]["type"].ToObject <string>();
foreach (KeyValuePair <string, string> pair in deploymentDict[linkedServiceName])
{
JToken token = serviceJObject.SelectToken(pair.Key);
token.Replace(pair.Value);
}
serviceJson = serviceJObject.ToString();
}
else
{
JObject serviceJObject = JObject.Parse(serviceJson);
linkedServiceType = serviceJObject["properties"]["type"].ToObject <string>();
}
// Search for Key Vault references in the linked service and replace with their Key Vault equivalents if found
if (adfFileHelper != null)
{
serviceJson = adfFileHelper.ResolveKeyVault(serviceJson).Result;
}
Models.LinkedService storageService;
switch (linkedServiceType)
{
case "AzureSqlDatabase":
case "OnPremisesSqlServer":
storageService = JsonConvert.DeserializeObject <Models.AzureSqlDatabaseLinkedService>(serviceJson);
break;
case "AzureStorage":
storageService = JsonConvert.DeserializeObject <Models.StorageService>(serviceJson);
break;
default:
throw new Exception($"Mapper for linked service type '{linkedServiceType}' not found.");
}
dummyServices.Add(storageService);
}
// parse the hd insight service json source
var computeServicePath = Path.Combine(adfFilesPath, dummyActivity.LinkedServiceName + ".json");
var computeServiceJson = File.ReadAllText(computeServicePath);
var computeService = JsonConvert.DeserializeObject <Models.ComputeService>(computeServiceJson);
dummyServices.Add(computeService);
// init the services
foreach (var dummyService in dummyServices)
{
LinkedService linkedService = null;
// init if it is a storage service
if (dummyService is Models.StorageService)
{
var dummyStorageService = dummyService as Models.StorageService;
var service = new AzureStorageLinkedService
{
ConnectionString = dummyStorageService.Properties.TypeProperties.ConnectionString
};
linkedService = new LinkedService(
dummyService.Name,
new LinkedServiceProperties(service)
);
}
// init if it is a AzureSqlDatabase service
if (dummyService is Models.AzureSqlDatabaseLinkedService)
{
var dummyStorageService = dummyService as Models.AzureSqlDatabaseLinkedService;
var service = new AzureSqlDatabaseLinkedService()
{
ConnectionString = dummyStorageService.Properties.TypeProperties.ConnectionString
};
linkedService = new LinkedService(
dummyService.Name,
new LinkedServiceProperties(service)
);
}
// init if it is a hd insight service
if (dummyService is Models.ComputeService)
{
var service = new HDInsightLinkedService();
linkedService = new LinkedService(
dummyService.Name,
new LinkedServiceProperties(service)
);
}
context.LinkedServices.Add(linkedService);
}
return(context);
}
