Esempio n. 1
0
        public static DotNetActivityContext DeserializeActivity(string pipelineFileName, string activityName, string configFile = null, string adfFilesPath = @"..\..\..\McioppDataFactory")
        {
            // Get Key Vault settings if secure publish is being used on the local machine
            AdfFileHelper adfFileHelper = null;
            string        settingsFile  = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "SecurePublishSettings.json");

            if (File.Exists(settingsFile))
            {
                AppSettings      settings = JsonConvert.DeserializeObject <AppSettings>(File.ReadAllText(settingsFile));
                X509Certificate2 cert     = KeyVaultResolver.FindCertificateByThumbprint(settings.KeyVaultCertThumbprint);
                string           suffix   = settings.EnvironmentSettings.First().KeyVaultDnsSuffix;
                suffix = string.IsNullOrEmpty(suffix) ? "vault.azure.net:443" : suffix;
                KeyVaultResolver keyVaultResolver = new KeyVaultResolver(settings.EnvironmentSettings.First().KeyVaultName, suffix, settings.KeyVaultCertClientId, cert);
                adfFileHelper = new AdfFileHelper(keyVaultResolver, new Logger());
            }

            adfFilesPath = Path.GetFullPath(adfFilesPath);

            var deploymentDict = new Dictionary <string, Dictionary <string, string> >();

            if (!string.IsNullOrEmpty(configFile))
            {
                // Get deployment config
                string deploymentConfigPath = Path.Combine(adfFilesPath, configFile);
                var    deploymentConfigJson = File.ReadAllText(deploymentConfigPath);
                var    deploymentJObj       = JObject.Parse(deploymentConfigJson);
                deploymentDict = deploymentJObj.Properties()
                                 .ToDictionary(x => x.Name,
                                               y => y.Value.ToDictionary(z => z["name"].ToString(), z => z["value"].ToString()));
            }

            DotNetActivityContext context = new DotNetActivityContext
            {
                LinkedServices = new List <LinkedService>(),
                Datasets       = new List <Dataset>(),
                Activity       = new Activity(),
                Logger         = new ActivityLogger()
            };

            string pipelinePath = Path.Combine(adfFilesPath, pipelineFileName);
            string pipelineJson = File.ReadAllText(pipelinePath);

            string pipelineName = Path.GetFileNameWithoutExtension(pipelineFileName);

            // Update with values from delpoyment config if exists
            if (deploymentDict.Count > 0 && deploymentDict.ContainsKey(pipelineName))
            {
                JObject pipelineJObject = JObject.Parse(pipelineJson);

                foreach (KeyValuePair <string, string> pair in deploymentDict[pipelineName])
                {
                    JToken token = pipelineJObject.SelectToken(pair.Key);
                    token.Replace(pair.Value);
                }

                pipelineJson = pipelineJObject.ToString();
            }

            // Search for Key Vault references in the pipeline and replace with their Key Vault equivalents if found
            if (adfFileHelper != null)
            {
                pipelineJson = adfFileHelper.ResolveKeyVault(pipelineJson).Result;
            }

            var dummyPipeline = JsonConvert.DeserializeObject <Models.Pipeline>(pipelineJson);

            Models.Activity dummyActivity;
            try
            {
                dummyActivity = dummyPipeline.Properties.Activities.Single(x => x.Name == activityName);
            }
            catch (InvalidOperationException)
            {
                throw new Exception($"Activity {activityName} not found in {pipelinePath}.");
            }

            context.Activity.Name = dummyActivity.Name;

            context.Activity.TypeProperties = new DotNetActivity();
            DotNetActivity dotNetActivity = (DotNetActivity)context.Activity.TypeProperties;

            dotNetActivity.ExtendedProperties = dummyActivity.DotNetActivityTypeProperties.ExtendedProperties;

            // get the input and output tables
            var dummyDatasets = new HashSet <Models.ActivityData>();

            dummyDatasets.UnionWith(dummyActivity.Inputs);
            dummyDatasets.UnionWith(dummyActivity.Outputs);

            var dummyServices = new HashSet <Models.LinkedService>();

            // init the data tables
            foreach (var dummyDataset in dummyDatasets)
            {
                // parse the table json source
                var dataPath   = Path.Combine(adfFilesPath, dummyDataset.Name + ".json");
                var dataJson   = File.ReadAllText(dataPath);
                var dummyTable = JsonConvert.DeserializeObject <Models.Table>(dataJson);
                {
                    // initialize dataset properties
                    DatasetTypeProperties datasetProperties;
                    switch (dummyTable.Properties.Type)
                    {
                    case "AzureBlob":
                        // init the azure model
                        var blobDataset = new AzureBlobDataset
                        {
                            FolderPath = dummyTable.Properties.TypeProperties.FolderPath,
                            FileName   = dummyTable.Properties.TypeProperties.FileName
                        };

                        datasetProperties = blobDataset;
                        break;

                    case "AzureTable":
                    case "AzureSqlTable":
                        var tableDataset = new AzureTableDataset
                        {
                            TableName = dummyTable.Properties.TypeProperties.TableName
                        };

                        datasetProperties = tableDataset;
                        break;

                    case "SqlServerTable":
                        var sqlTableDataset = new SqlServerTableDataset(dummyTable.Properties.TypeProperties.TableName);

                        datasetProperties = sqlTableDataset;
                        break;

                    default:
                        throw new Exception($"Unexpected Dataset.Type {dummyTable.Properties.Type}");
                    }

                    // initialize dataset

                    var dataDataset = new Dataset(
                        dummyDataset.Name,
                        new DatasetProperties(
                            datasetProperties,
                            new Availability(),
                            string.Empty
                            )
                        );

                    dataDataset.Properties.LinkedServiceName = dummyTable.Properties.LinkedServiceName;
                    context.Datasets.Add(dataDataset);
                }

                // register the inputs and outputs in the activity
                if (dummyDataset is Models.ActivityInput)
                {
                    context.Activity.Inputs.Add(new ActivityInput(dummyDataset.Name));
                }

                if (dummyDataset is Models.ActivityOutput)
                {
                    context.Activity.Outputs.Add(new ActivityOutput(dummyDataset.Name));
                }

                // parse the linked service json source for later use
                string linkedServiceName = dummyTable.Properties.LinkedServiceName;
                var    servicePath       = Path.Combine(adfFilesPath, linkedServiceName + ".json");
                string serviceJson       = File.ReadAllText(servicePath);

                string linkedServiceType = string.Empty;

                // Update with values from delpoyment config if exists
                if (deploymentDict.Count > 0 && deploymentDict.ContainsKey(linkedServiceName))
                {
                    JObject serviceJObject = JObject.Parse(serviceJson);
                    linkedServiceType = serviceJObject["properties"]["type"].ToObject <string>();

                    foreach (KeyValuePair <string, string> pair in deploymentDict[linkedServiceName])
                    {
                        JToken token = serviceJObject.SelectToken(pair.Key);
                        token.Replace(pair.Value);
                    }

                    serviceJson = serviceJObject.ToString();
                }
                else
                {
                    JObject serviceJObject = JObject.Parse(serviceJson);
                    linkedServiceType = serviceJObject["properties"]["type"].ToObject <string>();
                }

                // Search for Key Vault references in the linked service and replace with their Key Vault equivalents if found
                if (adfFileHelper != null)
                {
                    serviceJson = adfFileHelper.ResolveKeyVault(serviceJson).Result;
                }

                Models.LinkedService storageService;

                switch (linkedServiceType)
                {
                case "AzureSqlDatabase":
                case "OnPremisesSqlServer":
                    storageService = JsonConvert.DeserializeObject <Models.AzureSqlDatabaseLinkedService>(serviceJson);
                    break;

                case "AzureStorage":
                    storageService = JsonConvert.DeserializeObject <Models.StorageService>(serviceJson);
                    break;

                default:
                    throw new Exception($"Mapper for linked service type '{linkedServiceType}' not found.");
                }

                dummyServices.Add(storageService);
            }

            // parse the hd insight service json source
            var computeServicePath = Path.Combine(adfFilesPath, dummyActivity.LinkedServiceName + ".json");
            var computeServiceJson = File.ReadAllText(computeServicePath);
            var computeService     = JsonConvert.DeserializeObject <Models.ComputeService>(computeServiceJson);

            dummyServices.Add(computeService);


            // init the services
            foreach (var dummyService in dummyServices)
            {
                LinkedService linkedService = null;

                // init if it is a storage service
                if (dummyService is Models.StorageService)
                {
                    var dummyStorageService = dummyService as Models.StorageService;

                    var service = new AzureStorageLinkedService
                    {
                        ConnectionString = dummyStorageService.Properties.TypeProperties.ConnectionString
                    };

                    linkedService = new LinkedService(
                        dummyService.Name,
                        new LinkedServiceProperties(service)
                        );
                }

                // init if it is a AzureSqlDatabase service
                if (dummyService is Models.AzureSqlDatabaseLinkedService)
                {
                    var dummyStorageService = dummyService as Models.AzureSqlDatabaseLinkedService;

                    var service = new AzureSqlDatabaseLinkedService()
                    {
                        ConnectionString = dummyStorageService.Properties.TypeProperties.ConnectionString
                    };

                    linkedService = new LinkedService(
                        dummyService.Name,
                        new LinkedServiceProperties(service)
                        );
                }

                // init if it is a hd insight service
                if (dummyService is Models.ComputeService)
                {
                    var service = new HDInsightLinkedService();
                    linkedService = new LinkedService(
                        dummyService.Name,
                        new LinkedServiceProperties(service)
                        );
                }

                context.LinkedServices.Add(linkedService);
            }

            return(context);
        }
Esempio n. 2
0
        private async Task RefreshDatafactories()
        {
            SettingsContext settingsContext;

            environment = environmentList.SelectedItem as string;

            try
            {
                settingsContext = settingsContextManager.GetSettingsContext(environment);

                // Get KeyVault resolver which is used to retreive keyvault secrets based on environment context
                IKeyVaultResolver keyVault;
                if (!string.IsNullOrEmpty(settingsContext.KeyVaultCertificateThumbprint))
                {
                    var cert = KeyVaultResolver.FindCertificateByThumbprint(settingsContext.KeyVaultCertificateThumbprint);

                    if (cert == null)
                    {
                        Write($"No cert was found using thumbprint {settingsContext.KeyVaultCertificateThumbprint}", "Red");
                        return;
                    }

                    keyVault = new KeyVaultResolver(settingsContext.KeyVaultName, settingsContext.KeyVaultDnsSuffix, settingsContext.KeyVaultCertClientId, cert);
                }
                else
                {
                    keyVault = new KeyVaultResolver(settingsContext.KeyVaultName, settingsContext.KeyVaultDnsSuffix, settingsContext.KeyVaultCertClientId, settingsContext.KeyVaultCertClientSecret);
                }

                settingsContext.SubscriptionId = settings.Subscriptions[subscriptionList.SelectedIndex].Id;
                try
                {
                    settingsContext.AdfClientSecret = (await keyVault.GetSecret("SecurePublishAdfClientSecret")).Value;
                }
                catch (Exception ex)
                {
                    Write($"The secret called SecurePublishAdfClientSecret was not found in the KeyVault '{settingsContext.KeyVaultName}'. The ADF Client Secret is a password which was associated with the AAD Client ID '{settingsContext.KeyVaultCertClientId}' when it was originally set up. If you are setting up a new KeyVault and a previous KeyVault has already been used, you can get this value from the previous KeyVault. Otherwise refer to the user documentation on creating a new client ID and associating it with your Azure subscription.", "Red");
                    WriteError(ex);
                    return;
                }

                publishManager = new PublishManager(keyVault, settingsContext, this);

                dataFactoryList = await AzureAccessUtilities.GetDataFactories(settingsContext);
            }
            catch (Exception e)
            {
                Write(e.Message, "Red");

                Dispatcher.Invoke(() =>
                {
                    dataFactoryListBox.IsEnabled = false;
                });

                return;
            }

            if (!dataFactoryList.Any())
            {
                Write("No data factories found in subscription: " + settingsContext.SubscriptionId, "Orange");
                Write($"They either do not exist or else you may need to associate the Client ID '{settingsContext.AdfClientId}' with the subscription. To do that, perform the following steps:", "Orange");
                Write("1. Open up PowerShell", "Orange");
                Write("2. Log in to Azure by typing in the cmd: Login-AzureRmAccount", "Orange");
                Write($"3. Change to the subscription you wish to use by typing the cmd: Select-AzureRmSubscription -SubscriptionId '{settingsContext.SubscriptionId}'", "Orange");
                Write("4. Associate the Client ID with the Data Factory Contributer role in the current subscription by typing:", "Orange");
                Write($"New-AzureRmRoleAssignment -RoleDefinitionName 'Data Factory Contributor' -ServicePrincipalName '{settingsContext.AdfClientId}'", "Orange");

                Dispatcher.Invoke(() =>
                {
                    dataFactoryListBox.IsEnabled = false;
                });
            }
            else
            {
                Dispatcher.Invoke(() =>
                {
                    dataFactoryListBox.IsEnabled = true;
                });
            }

            Dispatcher.Invoke(() =>
            {
                dataFactoryListBox.ItemsSource = dataFactoryList.Select(x => x.Name);
            });
        }