public void CanSetNonGenericTypePropertiesWithoutTypeName() { var storageLinkedService = new AzureStorageLinkedService(); Assert.DoesNotThrow( () => { var ls = new LinkedServiceProperties(storageLinkedService); ls.TypeProperties = storageLinkedService; }); }
public override LinkedService ToSdkObject() { var linkedService = new AzureStorageLinkedService(); linkedService.ConnectionString = this.ConnectionString; linkedService.AccountKey = this.AccountKey; linkedService.SasUri = this.SasUri; linkedService.SasToken = this.SasToken; linkedService.EncryptedCredential = this.EncryptedCredential; SetProperties(linkedService); return(linkedService); }
/// <summary> /// Delete azure blob file or entire folder /// </summary> /// <param name="dataSetsToDelete"></param> /// public void DeleteBlobFileFolder(List <string> dataSetsToDelete) { foreach (string strInputToDelete in dataSetsToDelete) { Dataset inputDataset = datasets.First(ds => ds.Name.Equals(strInputToDelete)); AzureBlobDataset blobDataset = inputDataset.Properties.TypeProperties as AzureBlobDataset; logger.Write("\nBlob folder: " + blobDataset.FolderPath); logger.Write("\nBlob file: " + blobDataset.FileName); // linked service for input and output is the same. AzureStorageLinkedService linkedService = linkedServices.First(ls => ls.Name == inputDataset.Properties.LinkedServiceName).Properties.TypeProperties as AzureStorageLinkedService; // create storage client for input. Pass the connection string. CloudStorageAccount storageAccount = CloudStorageAccount.Parse(linkedService.ConnectionString); CloudBlobClient client = storageAccount.CreateCloudBlobClient(); // find blob to delete and delete if exists. Uri blobUri = new Uri(storageAccount.BlobEndpoint, blobDataset.FolderPath + blobDataset.FileName); CloudBlockBlob blob = new CloudBlockBlob(blobUri, storageAccount.Credentials); logger.Write("Blob Uri: {0}", blobUri.AbsoluteUri); logger.Write("Blob exists: {0}", blob.Exists()); blob.DeleteIfExists(); logger.Write("Deleted blob: {0}", blobUri.AbsoluteUri); // Ensure the container is exist. if (blobDataset.FolderPath.IndexOf("/") > 0) { string containerName = blobDataset.FolderPath.Substring(0, blobDataset.FolderPath.IndexOf("/")); logger.Write("Container Name {0}", containerName); string directoryName = blobDataset.FolderPath.Substring(blobDataset.FolderPath.IndexOf("/") + 1); logger.Write("Directory Name {0}", directoryName); var blobContainer = client.GetContainerReference(containerName); blobContainer.CreateIfNotExists(); CloudBlobDirectory cbd = blobContainer.GetDirectoryReference(directoryName); foreach (IListBlobItem item in blobContainer.ListBlobs(directoryName, true)) { logger.Write("Blob Uri: {0} ", item.Uri.AbsoluteUri); if (item.GetType() == typeof(CloudBlockBlob) || item.GetType().BaseType == typeof(CloudBlockBlob)) { CloudBlockBlob subBlob = new CloudBlockBlob(item.Uri, storageAccount.Credentials); logger.Write("Blob exists: {0}", subBlob.Exists()); subBlob.DeleteIfExists(); logger.Write("Deleted blob {0}", item.Uri.AbsoluteUri); } } } } }
// Get Linked Service Connection String private static string GetConnectionString(LinkedService asset) { if (asset == null) { return(null); } AzureStorageLinkedService storageAsset = asset.Properties as AzureStorageLinkedService; if (storageAsset == null) { return(null); } return(storageAsset.ConnectionString); }
private static void WriteDataToBlob( Dataset outputDataset, AzureStorageLinkedService outputLinkedService, IActivityLogger logger, string csvData ) { var folderPath = GetFolderPath(outputDataset); // create a storage object for the output blob. var outputConnectionString = outputLinkedService.ConnectionString; var outputStorageAccount = CloudStorageAccount.Parse(outputConnectionString); var outputBlobUri = new Uri(outputStorageAccount.BlobEndpoint, folderPath + "/" + GetFileName(outputDataset)); var outputBlob = new CloudBlockBlob(outputBlobUri, outputStorageAccount.Credentials); outputBlob.UploadText(csvData); }
public async Task <ActionResult <LinkedServiceResource> > AddDataSource2Async(Guid engineId, string dataSourceName, [FromBody] LinkedService linkedService) { var engine = await this.engineProvider.GetEngineAsync(engineId).ConfigureAwait(false); if (engine == null) { throw new Exception("Engine does not exists"); } var accessToken = await this.authProvider.GetAccessTokenForAppManagementAsync().ConfigureAwait(false); ServiceClientCredentials tokenCredentials = new TokenCredentials(accessToken); DataFactoryManagementClient client = new DataFactoryManagementClient(tokenCredentials) { SubscriptionId = this.options.SubscriptionId }; // Create the linked service resource LinkedServiceResource linkedServiceResource = new LinkedServiceResource(linkedService); string sensitiveString = linkedServiceResource.Properties switch { AzureStorageLinkedService azureStorageLinkedService => azureStorageLinkedService.ConnectionString.ToString(), _ => null }; if (!string.IsNullOrEmpty(sensitiveString)) { // Save the connection string to KeyVault await keyVaultsController.SetKeyVaultSecret(engineId, dataSourceName, new YKeyVaultSecretPayload { Key = dataSourceName, Value = sensitiveString }); } var newLinkedServiceResourceCreated = await client.LinkedServices.CreateOrUpdateAsync(engine.ResourceGroupName, engine.FactoryName, dataSourceName, linkedServiceResource); return(newLinkedServiceResourceCreated); }
public IDictionary <string, string> Execute(IEnumerable <LinkedService> linkedServices, IEnumerable <Dataset> datasets, Activity activity, IActivityLogger logger) { try { logger.Write("Custom Activity Started."); DotNetActivity dotNetActivity = (DotNetActivity)activity.TypeProperties; string inputToDelete = dotNetActivity.ExtendedProperties["InputToDelete"]; logger.Write("\nInput to delete is " + inputToDelete); logger.Write("\nAll Dataset(s) Below "); foreach (Dataset ds in datasets) { logger.Write("\nDataset: " + ds.Name); } foreach (string name in activity.Inputs.Select(i => i.Name)) { logger.Write("\nInput Dataset: " + name); } foreach (string name in activity.Outputs.Select(i => i.Name)) { logger.Write("\nOutput Dataset: " + name); } List <string> dataSetsToDelete = inputToDelete.Split(',').ToList(); foreach (string strInputToDelete in dataSetsToDelete) { Dataset inputDataset = datasets.First(ds => ds.Name.Equals(strInputToDelete)); AzureBlobDataset blobDataset = inputDataset.Properties.TypeProperties as AzureBlobDataset; logger.Write("\nBlob folder: " + blobDataset.FolderPath); logger.Write("\nBlob file: " + blobDataset.FileName); // linked service for input and output is the same. AzureStorageLinkedService linkedService = linkedServices.First(ls => ls.Name == inputDataset.Properties.LinkedServiceName).Properties.TypeProperties as AzureStorageLinkedService; // create storage client for input. Pass the connection string. CloudStorageAccount storageAccount = CloudStorageAccount.Parse(linkedService.ConnectionString); CloudBlobClient client = storageAccount.CreateCloudBlobClient(); // find blob to delete and delete if exists. Uri blobUri = new Uri(storageAccount.BlobEndpoint, blobDataset.FolderPath + blobDataset.FileName); CloudBlockBlob blob = new CloudBlockBlob(blobUri, storageAccount.Credentials); logger.Write("Blob Uri: {0}", blobUri.AbsoluteUri); logger.Write("Blob exists: {0}", blob.Exists()); blob.DeleteIfExists(); logger.Write("Deleted blob: {0}", blobUri.AbsoluteUri); // Ensure the container is exist. if (blobDataset.FolderPath.IndexOf("/") > 0) { string containerName = blobDataset.FolderPath.Substring(0, blobDataset.FolderPath.IndexOf("/")); logger.Write("Container Name {0}", containerName); string directoryName = blobDataset.FolderPath.Substring(blobDataset.FolderPath.IndexOf("/") + 1); logger.Write("Directory Name {0}", directoryName); var blobContainer = client.GetContainerReference(containerName); blobContainer.CreateIfNotExists(); CloudBlobDirectory cbd = blobContainer.GetDirectoryReference(directoryName); foreach (IListBlobItem item in blobContainer.ListBlobs(directoryName, true)) { logger.Write("Blob Uri: {0} ", item.Uri.AbsoluteUri); if (item.GetType() == typeof(CloudBlockBlob) || item.GetType().BaseType == typeof(CloudBlockBlob)) { CloudBlockBlob subBlob = new CloudBlockBlob(item.Uri, storageAccount.Credentials); logger.Write("Blob exists: {0}", subBlob.Exists()); subBlob.DeleteIfExists(); logger.Write("Deleted blob {0}", item.Uri.AbsoluteUri); } } } } logger.Write("Custom Activity Ended Successfully."); } catch (Exception e) { logger.Write("Custom Activity Failed with error."); logger.Write("Caught exception: "); logger.Write(e.Message); throw new Exception(e.Message); } // The dictionary can be used to chain custom activities together in the future. // This feature is not implemented yet, so just return an empty dictionary. return(new Dictionary <string, string>()); }
public static DotNetActivityContext DeserializeActivity(string pipelineFileName, string activityName, string configFile = null, string adfFilesPath = @"..\..\..\McioppDataFactory") { // Get Key Vault settings if secure publish is being used on the local machine AdfFileHelper adfFileHelper = null; string settingsFile = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "SecurePublishSettings.json"); if (File.Exists(settingsFile)) { AppSettings settings = JsonConvert.DeserializeObject <AppSettings>(File.ReadAllText(settingsFile)); X509Certificate2 cert = KeyVaultResolver.FindCertificateByThumbprint(settings.KeyVaultCertThumbprint); string suffix = settings.EnvironmentSettings.First().KeyVaultDnsSuffix; suffix = string.IsNullOrEmpty(suffix) ? "vault.azure.net:443" : suffix; KeyVaultResolver keyVaultResolver = new KeyVaultResolver(settings.EnvironmentSettings.First().KeyVaultName, suffix, settings.KeyVaultCertClientId, cert); adfFileHelper = new AdfFileHelper(keyVaultResolver, new Logger()); } adfFilesPath = Path.GetFullPath(adfFilesPath); var deploymentDict = new Dictionary <string, Dictionary <string, string> >(); if (!string.IsNullOrEmpty(configFile)) { // Get deployment config string deploymentConfigPath = Path.Combine(adfFilesPath, configFile); var deploymentConfigJson = File.ReadAllText(deploymentConfigPath); var deploymentJObj = JObject.Parse(deploymentConfigJson); deploymentDict = deploymentJObj.Properties() .ToDictionary(x => x.Name, y => y.Value.ToDictionary(z => z["name"].ToString(), z => z["value"].ToString())); } DotNetActivityContext context = new DotNetActivityContext { LinkedServices = new List <LinkedService>(), Datasets = new List <Dataset>(), Activity = new Activity(), Logger = new ActivityLogger() }; string pipelinePath = Path.Combine(adfFilesPath, pipelineFileName); string pipelineJson = File.ReadAllText(pipelinePath); string pipelineName = Path.GetFileNameWithoutExtension(pipelineFileName); // Update with values from delpoyment config if exists if (deploymentDict.Count > 0 && deploymentDict.ContainsKey(pipelineName)) { JObject pipelineJObject = JObject.Parse(pipelineJson); foreach (KeyValuePair <string, string> pair in deploymentDict[pipelineName]) { JToken token = pipelineJObject.SelectToken(pair.Key); token.Replace(pair.Value); } pipelineJson = pipelineJObject.ToString(); } // Search for Key Vault references in the pipeline and replace with their Key Vault equivalents if found if (adfFileHelper != null) { pipelineJson = adfFileHelper.ResolveKeyVault(pipelineJson).Result; } var dummyPipeline = JsonConvert.DeserializeObject <Models.Pipeline>(pipelineJson); Models.Activity dummyActivity; try { dummyActivity = dummyPipeline.Properties.Activities.Single(x => x.Name == activityName); } catch (InvalidOperationException) { throw new Exception($"Activity {activityName} not found in {pipelinePath}."); } context.Activity.Name = dummyActivity.Name; context.Activity.TypeProperties = new DotNetActivity(); DotNetActivity dotNetActivity = (DotNetActivity)context.Activity.TypeProperties; dotNetActivity.ExtendedProperties = dummyActivity.DotNetActivityTypeProperties.ExtendedProperties; // get the input and output tables var dummyDatasets = new HashSet <Models.ActivityData>(); dummyDatasets.UnionWith(dummyActivity.Inputs); dummyDatasets.UnionWith(dummyActivity.Outputs); var dummyServices = new HashSet <Models.LinkedService>(); // init the data tables foreach (var dummyDataset in dummyDatasets) { // parse the table json source var dataPath = Path.Combine(adfFilesPath, dummyDataset.Name + ".json"); var dataJson = File.ReadAllText(dataPath); var dummyTable = JsonConvert.DeserializeObject <Models.Table>(dataJson); { // initialize dataset properties DatasetTypeProperties datasetProperties; switch (dummyTable.Properties.Type) { case "AzureBlob": // init the azure model var blobDataset = new AzureBlobDataset { FolderPath = dummyTable.Properties.TypeProperties.FolderPath, FileName = dummyTable.Properties.TypeProperties.FileName }; datasetProperties = blobDataset; break; case "AzureTable": case "AzureSqlTable": var tableDataset = new AzureTableDataset { TableName = dummyTable.Properties.TypeProperties.TableName }; datasetProperties = tableDataset; break; case "SqlServerTable": var sqlTableDataset = new SqlServerTableDataset(dummyTable.Properties.TypeProperties.TableName); datasetProperties = sqlTableDataset; break; default: throw new Exception($"Unexpected Dataset.Type {dummyTable.Properties.Type}"); } // initialize dataset var dataDataset = new Dataset( dummyDataset.Name, new DatasetProperties( datasetProperties, new Availability(), string.Empty ) ); dataDataset.Properties.LinkedServiceName = dummyTable.Properties.LinkedServiceName; context.Datasets.Add(dataDataset); } // register the inputs and outputs in the activity if (dummyDataset is Models.ActivityInput) { context.Activity.Inputs.Add(new ActivityInput(dummyDataset.Name)); } if (dummyDataset is Models.ActivityOutput) { context.Activity.Outputs.Add(new ActivityOutput(dummyDataset.Name)); } // parse the linked service json source for later use string linkedServiceName = dummyTable.Properties.LinkedServiceName; var servicePath = Path.Combine(adfFilesPath, linkedServiceName + ".json"); string serviceJson = File.ReadAllText(servicePath); string linkedServiceType = string.Empty; // Update with values from delpoyment config if exists if (deploymentDict.Count > 0 && deploymentDict.ContainsKey(linkedServiceName)) { JObject serviceJObject = JObject.Parse(serviceJson); linkedServiceType = serviceJObject["properties"]["type"].ToObject <string>(); foreach (KeyValuePair <string, string> pair in deploymentDict[linkedServiceName]) { JToken token = serviceJObject.SelectToken(pair.Key); token.Replace(pair.Value); } serviceJson = serviceJObject.ToString(); } else { JObject serviceJObject = JObject.Parse(serviceJson); linkedServiceType = serviceJObject["properties"]["type"].ToObject <string>(); } // Search for Key Vault references in the linked service and replace with their Key Vault equivalents if found if (adfFileHelper != null) { serviceJson = adfFileHelper.ResolveKeyVault(serviceJson).Result; } Models.LinkedService storageService; switch (linkedServiceType) { case "AzureSqlDatabase": case "OnPremisesSqlServer": storageService = JsonConvert.DeserializeObject <Models.AzureSqlDatabaseLinkedService>(serviceJson); break; case "AzureStorage": storageService = JsonConvert.DeserializeObject <Models.StorageService>(serviceJson); break; default: throw new Exception($"Mapper for linked service type '{linkedServiceType}' not found."); } dummyServices.Add(storageService); } // parse the hd insight service json source var computeServicePath = Path.Combine(adfFilesPath, dummyActivity.LinkedServiceName + ".json"); var computeServiceJson = File.ReadAllText(computeServicePath); var computeService = JsonConvert.DeserializeObject <Models.ComputeService>(computeServiceJson); dummyServices.Add(computeService); // init the services foreach (var dummyService in dummyServices) { LinkedService linkedService = null; // init if it is a storage service if (dummyService is Models.StorageService) { var dummyStorageService = dummyService as Models.StorageService; var service = new AzureStorageLinkedService { ConnectionString = dummyStorageService.Properties.TypeProperties.ConnectionString }; linkedService = new LinkedService( dummyService.Name, new LinkedServiceProperties(service) ); } // init if it is a AzureSqlDatabase service if (dummyService is Models.AzureSqlDatabaseLinkedService) { var dummyStorageService = dummyService as Models.AzureSqlDatabaseLinkedService; var service = new AzureSqlDatabaseLinkedService() { ConnectionString = dummyStorageService.Properties.TypeProperties.ConnectionString }; linkedService = new LinkedService( dummyService.Name, new LinkedServiceProperties(service) ); } // init if it is a hd insight service if (dummyService is Models.ComputeService) { var service = new HDInsightLinkedService(); linkedService = new LinkedService( dummyService.Name, new LinkedServiceProperties(service) ); } context.LinkedServices.Add(linkedService); } return(context); }