public static DotNetActivityContext DeserializeActivity(string pipelineFileName, string activityName, string configFile = null, string adfFilesPath = @"..\..\..\McioppDataFactory")
{
// Get Key Vault settings if secure publish is being used on the local machine
AdfFileHelper adfFileHelper = null;
string settingsFile = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "SecurePublishSettings.json");
if (File.Exists(settingsFile))
{
AppSettings settings = JsonConvert.DeserializeObject <AppSettings>(File.ReadAllText(settingsFile));
X509Certificate2 cert = KeyVaultResolver.FindCertificateByThumbprint(settings.KeyVaultCertThumbprint);
string suffix = settings.EnvironmentSettings.First().KeyVaultDnsSuffix;
suffix = string.IsNullOrEmpty(suffix) ? "vault.azure.net:443" : suffix;
KeyVaultResolver keyVaultResolver = new KeyVaultResolver(settings.EnvironmentSettings.First().KeyVaultName, suffix, settings.KeyVaultCertClientId, cert);
adfFileHelper = new AdfFileHelper(keyVaultResolver, new Logger());
}
adfFilesPath = Path.GetFullPath(adfFilesPath);
var deploymentDict = new Dictionary <string, Dictionary <string, string> >();
if (!string.IsNullOrEmpty(configFile))
{
// Get deployment config
string deploymentConfigPath = Path.Combine(adfFilesPath, configFile);
var deploymentConfigJson = File.ReadAllText(deploymentConfigPath);
var deploymentJObj = JObject.Parse(deploymentConfigJson);
deploymentDict = deploymentJObj.Properties()
.ToDictionary(x => x.Name,
y => y.Value.ToDictionary(z => z["name"].ToString(), z => z["value"].ToString()));
}
DotNetActivityContext context = new DotNetActivityContext
{
LinkedServices = new List <LinkedService>(),
Datasets = new List <Dataset>(),
Activity = new Activity(),
Logger = new ActivityLogger()
};
string pipelinePath = Path.Combine(adfFilesPath, pipelineFileName);
string pipelineJson = File.ReadAllText(pipelinePath);
string pipelineName = Path.GetFileNameWithoutExtension(pipelineFileName);
// Update with values from delpoyment config if exists
if (deploymentDict.Count > 0 && deploymentDict.ContainsKey(pipelineName))
{
JObject pipelineJObject = JObject.Parse(pipelineJson);
foreach (KeyValuePair <string, string> pair in deploymentDict[pipelineName])
{
JToken token = pipelineJObject.SelectToken(pair.Key);
token.Replace(pair.Value);
}
pipelineJson = pipelineJObject.ToString();
}
// Search for Key Vault references in the pipeline and replace with their Key Vault equivalents if found
if (adfFileHelper != null)
{
pipelineJson = adfFileHelper.ResolveKeyVault(pipelineJson).Result;
}
var dummyPipeline = JsonConvert.DeserializeObject <Models.Pipeline>(pipelineJson);
Models.Activity dummyActivity;
try
{
dummyActivity = dummyPipeline.Properties.Activities.Single(x => x.Name == activityName);
}
catch (InvalidOperationException)
{
throw new Exception($"Activity {activityName} not found in {pipelinePath}.");
}
context.Activity.Name = dummyActivity.Name;
context.Activity.TypeProperties = new DotNetActivity();
DotNetActivity dotNetActivity = (DotNetActivity)context.Activity.TypeProperties;
dotNetActivity.ExtendedProperties = dummyActivity.DotNetActivityTypeProperties.ExtendedProperties;
// get the input and output tables
var dummyDatasets = new HashSet <Models.ActivityData>();
dummyDatasets.UnionWith(dummyActivity.Inputs);
dummyDatasets.UnionWith(dummyActivity.Outputs);
var dummyServices = new HashSet <Models.LinkedService>();
// init the data tables
foreach (var dummyDataset in dummyDatasets)
{
// parse the table json source
var dataPath = Path.Combine(adfFilesPath, dummyDataset.Name + ".json");
var dataJson = File.ReadAllText(dataPath);
var dummyTable = JsonConvert.DeserializeObject <Models.Table>(dataJson);
{
// initialize dataset properties
DatasetTypeProperties datasetProperties;
switch (dummyTable.Properties.Type)
{
case "AzureBlob":
// init the azure model
var blobDataset = new AzureBlobDataset
{
FolderPath = dummyTable.Properties.TypeProperties.FolderPath,
FileName = dummyTable.Properties.TypeProperties.FileName
};
datasetProperties = blobDataset;
break;
case "AzureTable":
case "AzureSqlTable":
var tableDataset = new AzureTableDataset
{
TableName = dummyTable.Properties.TypeProperties.TableName
};
datasetProperties = tableDataset;
break;
case "SqlServerTable":
var sqlTableDataset = new SqlServerTableDataset(dummyTable.Properties.TypeProperties.TableName);
datasetProperties = sqlTableDataset;
break;
default:
throw new Exception($"Unexpected Dataset.Type {dummyTable.Properties.Type}");
}
// initialize dataset
var dataDataset = new Dataset(
dummyDataset.Name,
new DatasetProperties(
datasetProperties,
new Availability(),
string.Empty
)
);
dataDataset.Properties.LinkedServiceName = dummyTable.Properties.LinkedServiceName;
context.Datasets.Add(dataDataset);
}
// register the inputs and outputs in the activity
if (dummyDataset is Models.ActivityInput)
{
context.Activity.Inputs.Add(new ActivityInput(dummyDataset.Name));
}
if (dummyDataset is Models.ActivityOutput)
{
context.Activity.Outputs.Add(new ActivityOutput(dummyDataset.Name));
}
// parse the linked service json source for later use
string linkedServiceName = dummyTable.Properties.LinkedServiceName;
var servicePath = Path.Combine(adfFilesPath, linkedServiceName + ".json");
string serviceJson = File.ReadAllText(servicePath);
string linkedServiceType = string.Empty;
// Update with values from delpoyment config if exists
if (deploymentDict.Count > 0 && deploymentDict.ContainsKey(linkedServiceName))
{
JObject serviceJObject = JObject.Parse(serviceJson);
linkedServiceType = serviceJObject["properties"]["type"].ToObject <string>();
foreach (KeyValuePair <string, string> pair in deploymentDict[linkedServiceName])
{
JToken token = serviceJObject.SelectToken(pair.Key);
token.Replace(pair.Value);
}
serviceJson = serviceJObject.ToString();
}
else
{
JObject serviceJObject = JObject.Parse(serviceJson);
linkedServiceType = serviceJObject["properties"]["type"].ToObject <string>();
}
// Search for Key Vault references in the linked service and replace with their Key Vault equivalents if found
if (adfFileHelper != null)
{
serviceJson = adfFileHelper.ResolveKeyVault(serviceJson).Result;
}
Models.LinkedService storageService;
switch (linkedServiceType)
{
case "AzureSqlDatabase":
case "OnPremisesSqlServer":
storageService = JsonConvert.DeserializeObject <Models.AzureSqlDatabaseLinkedService>(serviceJson);
break;
case "AzureStorage":
storageService = JsonConvert.DeserializeObject <Models.StorageService>(serviceJson);
break;
default:
throw new Exception($"Mapper for linked service type '{linkedServiceType}' not found.");
}
dummyServices.Add(storageService);
}
// parse the hd insight service json source
var computeServicePath = Path.Combine(adfFilesPath, dummyActivity.LinkedServiceName + ".json");
var computeServiceJson = File.ReadAllText(computeServicePath);
var computeService = JsonConvert.DeserializeObject <Models.ComputeService>(computeServiceJson);
dummyServices.Add(computeService);
// init the services
foreach (var dummyService in dummyServices)
{
LinkedService linkedService = null;
// init if it is a storage service
if (dummyService is Models.StorageService)
{
var dummyStorageService = dummyService as Models.StorageService;
var service = new AzureStorageLinkedService
{
ConnectionString = dummyStorageService.Properties.TypeProperties.ConnectionString
};
linkedService = new LinkedService(
dummyService.Name,
new LinkedServiceProperties(service)
);
}
// init if it is a AzureSqlDatabase service
if (dummyService is Models.AzureSqlDatabaseLinkedService)
{
var dummyStorageService = dummyService as Models.AzureSqlDatabaseLinkedService;
var service = new AzureSqlDatabaseLinkedService()
{
ConnectionString = dummyStorageService.Properties.TypeProperties.ConnectionString
};
linkedService = new LinkedService(
dummyService.Name,
new LinkedServiceProperties(service)
);
}
// init if it is a hd insight service
if (dummyService is Models.ComputeService)
{
var service = new HDInsightLinkedService();
linkedService = new LinkedService(
dummyService.Name,
new LinkedServiceProperties(service)
);
}
context.LinkedServices.Add(linkedService);
}
return(context);
}
private async Task RefreshDatafactories()
{
SettingsContext settingsContext;
environment = environmentList.SelectedItem as string;
try
{
settingsContext = settingsContextManager.GetSettingsContext(environment);
// Get KeyVault resolver which is used to retreive keyvault secrets based on environment context
IKeyVaultResolver keyVault;
if (!string.IsNullOrEmpty(settingsContext.KeyVaultCertificateThumbprint))
{
var cert = KeyVaultResolver.FindCertificateByThumbprint(settingsContext.KeyVaultCertificateThumbprint);
if (cert == null)
{
Write($"No cert was found using thumbprint {settingsContext.KeyVaultCertificateThumbprint}", "Red");
return;
}
keyVault = new KeyVaultResolver(settingsContext.KeyVaultName, settingsContext.KeyVaultDnsSuffix, settingsContext.KeyVaultCertClientId, cert);
}
else
{
keyVault = new KeyVaultResolver(settingsContext.KeyVaultName, settingsContext.KeyVaultDnsSuffix, settingsContext.KeyVaultCertClientId, settingsContext.KeyVaultCertClientSecret);
}
settingsContext.SubscriptionId = settings.Subscriptions[subscriptionList.SelectedIndex].Id;
try
{
settingsContext.AdfClientSecret = (await keyVault.GetSecret("SecurePublishAdfClientSecret")).Value;
}
catch (Exception ex)
{
Write($"The secret called SecurePublishAdfClientSecret was not found in the KeyVault '{settingsContext.KeyVaultName}'. The ADF Client Secret is a password which was associated with the AAD Client ID '{settingsContext.KeyVaultCertClientId}' when it was originally set up. If you are setting up a new KeyVault and a previous KeyVault has already been used, you can get this value from the previous KeyVault. Otherwise refer to the user documentation on creating a new client ID and associating it with your Azure subscription.", "Red");
WriteError(ex);
return;
}
publishManager = new PublishManager(keyVault, settingsContext, this);
dataFactoryList = await AzureAccessUtilities.GetDataFactories(settingsContext);
}
catch (Exception e)
{
Write(e.Message, "Red");
Dispatcher.Invoke(() =>
{
dataFactoryListBox.IsEnabled = false;
});
return;
}
if (!dataFactoryList.Any())
{
Write("No data factories found in subscription: " + settingsContext.SubscriptionId, "Orange");
Write($"They either do not exist or else you may need to associate the Client ID '{settingsContext.AdfClientId}' with the subscription. To do that, perform the following steps:", "Orange");
Write("1. Open up PowerShell", "Orange");
Write("2. Log in to Azure by typing in the cmd: Login-AzureRmAccount", "Orange");
Write($"3. Change to the subscription you wish to use by typing the cmd: Select-AzureRmSubscription -SubscriptionId '{settingsContext.SubscriptionId}'", "Orange");
Write("4. Associate the Client ID with the Data Factory Contributer role in the current subscription by typing:", "Orange");
Write($"New-AzureRmRoleAssignment -RoleDefinitionName 'Data Factory Contributor' -ServicePrincipalName '{settingsContext.AdfClientId}'", "Orange");
Dispatcher.Invoke(() =>
{
dataFactoryListBox.IsEnabled = false;
});
}
else
{
Dispatcher.Invoke(() =>
{
dataFactoryListBox.IsEnabled = true;
});
}
Dispatcher.Invoke(() =>
{
dataFactoryListBox.ItemsSource = dataFactoryList.Select(x => x.Name);
});
}
