private static Dictionary <string, DeployConfigInfo> deployConfigs; //服务器列表 public static Dictionary <string, DeployConfigInfo> GetDeployConfigInfos() { if (deployConfigs == null) { string jsonstr = "";// SingletonMB<ResourceManagerController>.GetInstance().LoadData(AssetConst.JsonData, "deploy"); JsonData jd = JsonMapper.ToObject(jsonstr); JsonData items = jd["items"]; Dictionary <string, DeployConfigInfo> cinfos = new Dictionary <string, DeployConfigInfo>(); int count = items.Count; for (int i = 0; i < count; i++) { JsonData item = items[i]; string json = JsonMapper.ToJson(item); DeployConfigInfo cinfo = GetConfigInfo("deploy", json) as DeployConfigInfo; if (cinfo != null) { if (!cinfos.ContainsKey(cinfo.param)) { cinfos.Add(cinfo.param, cinfo); } } } deployConfigs = cinfos; } return(deployConfigs); }
/// <summary> /// Publishes to the specified data factory given a prebuilt output folder /// </summary> /// <returns>True if the publish succeeds othewise false</returns> public async Task <bool> PublishFromOutputFolder(string adfOutputFolder, string dataFactoryResourceGroup, string dataFactoryName) { try { // Get schemas used for determining ADF files await adfFileHelper.GetSchemas(httpClient); List <string> filesToProcess = Directory.GetFiles(adfOutputFolder, "*.json", SearchOption.TopDirectoryOnly).ToList(); var lowerExcludeList = settingsContext.FilesToExclude == null || !settingsContext.FilesToExclude.Any() ? new List <string>() : settingsContext.FilesToExclude.Select(x => x.ToLowerInvariant()); filesToProcess = filesToProcess.Where(x => { var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(x); return(fileNameWithoutExtension != null && !lowerExcludeList.Contains(fileNameWithoutExtension.ToLowerInvariant())); }).ToList(); string deployConfigPath = string.IsNullOrEmpty(settingsContext.DeploymentConfigName) ? null : filesToProcess.FirstOrDefault(x => { var fileName = Path.GetFileName(x); return(fileName != null && fileName.Equals(Path.GetFileNameWithoutExtension(settingsContext.DeploymentConfigName) + ".json", StringComparison.InvariantCultureIgnoreCase)); }); DeployConfigInfo deployConfig = null; logger.Write(string.Empty); if (!string.IsNullOrEmpty(deployConfigPath)) { logger.Write("Using deployment configuration file: " + deployConfigPath); deployConfig = adfFileHelper.GetDeployConfigInfo(deployConfigPath); } else { logger.Write("No deployment configuration file found.", "Orange"); } AdfDeploy adf = new AdfDeploy(adfFileHelper, logger, blob, settingsContext, dataFactoryResourceGroup, dataFactoryName); return(await adf.Deploy(filesToProcess, adfOutputFolder, deployConfig)); } catch (Exception e) { logger.Write($"Error: {e.Message}"); logger.WriteError(e); return(false); } }
/// <summary> /// Resolves the deployment settings from the deployment config. /// </summary> private JObject ResolveDeploymentSettings(JObject jObject, string name, DeployConfigInfo deployConfig) { // Update with values from delpoyment config if exists if (deployConfig.DeploymentDictionary.Count > 0 && deployConfig.DeploymentDictionary.ContainsKey(name)) { foreach (KeyValuePair <string, string> pair in deployConfig.DeploymentDictionary[name]) { JToken token = jObject.SelectToken(pair.Key); if (token == null) { logger.Write( $"The deployment configuration file '{deployConfig.FileName}' contains a substitution for '{name}' however the JPath '{pair.Key}' is not found in '{name}'."); } else { token.Replace(pair.Value); } } } return(jObject); }
/// <summary> /// Gets the information on the ADF file. /// </summary> /// <param name="filePath">The file path.</param> /// <param name="deployConfig">The deploy configuration.</param> public async Task <AdfFileInfo> GetFileInfo(string filePath, DeployConfigInfo deployConfig = null) { string fileContents = File.ReadAllText(filePath); string fileName = Path.GetFileNameWithoutExtension(filePath); // Initialize props to default values AdfFileInfo fileInfo = new AdfFileInfo { FileType = FileType.Unknown, SubType = string.Empty, Name = string.Empty, IsValid = false, FileContents = fileContents, FileName = fileName }; JObject jObject = null; try { jObject = JObject.Parse(fileContents); JObject propertyJObject = jObject.GetValue("properties", StringComparison.OrdinalIgnoreCase) as JObject; JToken nameJToken = jObject.GetValue("name", StringComparison.OrdinalIgnoreCase); if (propertyJObject == null || nameJToken == null) { logger.Write($"{fileInfo.FileName} is a not a valid ADF file."); return(fileInfo); } fileInfo.Name = nameJToken.ToObject <string>(); if (deployConfig != null) { // Update ADF files with deploymnet settings if they exist jObject = ResolveDeploymentSettings(jObject, fileInfo.Name, deployConfig); fileInfo.FileContents = jObject.ToString(); } JToken typeJToken = propertyJObject.GetValue("type", StringComparison.OrdinalIgnoreCase); if (typeJToken != null) { fileInfo.SubType = typeJToken.ToObject <string>(); } if (fileInfo.SubType == "CPSServiceBusProxy" || jObject.IsValid(jsonLinkedServiceSchema)) { fileInfo.FileType = FileType.LinkedService; fileInfo.IsValid = true; logger.Write($"Retreived LinkedService: {fileInfo.FileName}"); } else if (jObject.IsValid(jsonTableSchema)) { fileInfo.FileType = FileType.Table; fileInfo.IsValid = true; logger.Write($"Retreived Dataset: {fileInfo.FileName}"); } else if (jObject.IsValid(jsonPipelineSchema)) { fileInfo.FileType = FileType.Pipeline; fileInfo.IsValid = true; logger.Write($"Retreived Pipeline: {fileInfo.FileName}"); // Get all custom activity packages if available JArray activities = propertyJObject.GetValue("activities", StringComparison.InvariantCultureIgnoreCase) as JArray; if (activities != null) { fileInfo.CustomActivityPackages = new List <CustomActivityPackageInfo>(); foreach (JObject activity in activities) { JToken activityTypeJToken = activity.GetValue("type", StringComparison.OrdinalIgnoreCase); if (activityTypeJToken != null && activityTypeJToken.ToObject <string>().Equals("DotNetActivity", StringComparison.CurrentCultureIgnoreCase)) { CustomActivityPackageInfo packageInfo = new CustomActivityPackageInfo(); packageInfo.PackageLinkedService = activity.SelectToken("$.typeProperties.packageLinkedService")?.ToObject <string>(); packageInfo.PackageFile = activity.SelectToken("$.typeProperties.packageFile")?.ToObject <string>(); logger.Write($"Retreived Custom Activity package: {packageInfo.PackageFile}"); fileInfo.CustomActivityPackages.Add(packageInfo); } } } } else { fileInfo.FileType = FileType.Unknown; logger.Write($"{fileInfo.FileName} is a not a valid ADF file."); } } catch (Exception e) { fileInfo.ErrorException = e; logger.Write($"{fileInfo.FileName} is a not a valid ADF file. Error message: {e}"); logger.WriteError(e); } if (fileInfo.IsValid) { // Search for keyvault tokens and resolve them string keyVaultResolvedContents = await ResolveKeyVault(fileInfo.FileContents); if (fileInfo.FileContents != keyVaultResolvedContents) { fileInfo.FileContents = keyVaultResolvedContents; fileInfo.JObject = JObject.Parse(fileInfo.FileContents); } else { fileInfo.JObject = jObject; } } return(fileInfo); }
/// <summary> /// Deploys the specified ADF files and custom activity packages to Azure. /// </summary> /// <param name="filesToProcess">The files to process.</param> /// <param name="outputFolder">The output folder which contains the files and custom activity zips.</param> /// <param name="deployConfig">The deployment configuration information.</param> public async Task <bool> Deploy(List <string> filesToProcess, string outputFolder, DeployConfigInfo deployConfig = null) { bool result = true; logger.Write(string.Empty); logger.Write($"Getting all ADF resources to deploy to Azure from output folder '{outputFolder}'", "Black"); List <Task <AdfFileInfo> > allFilesTasks = filesToProcess.Select(async x => await adfFileHelper.GetFileInfo(x, deployConfig)).ToList(); List <AdfFileInfo> allFiles = new List <AdfFileInfo>(); foreach (var allFilesTask in allFilesTasks) { allFiles.Add(await allFilesTask); } List <AdfFileInfo> validFiles = allFiles.Where(x => x.IsValid).ToList(); if (!validFiles.Any()) { logger.Write($"No valid ADF files found in '{outputFolder}'", "Red"); return(false); } logger.Write($"{validFiles.Count} file{(validFiles.Count == 1 ? string.Empty : "s")} retreived"); logger.Write(string.Empty); logger.Write($"Begin deploying ADF resources to '{dataFactoryName}'", "Black"); // Log invalid files List <AdfFileInfo> invalidFiles = allFiles.Where(x => !x.IsValid).ToList(); if (invalidFiles.Any()) { logger.Write("The following files found in the output folder will not be published:"); foreach (AdfFileInfo invalidFile in invalidFiles) { logger.Write(invalidFile.FileName); } } DataFactoryManagementClient client = GetDataFactoryManagementClient(); // Deploy Package Zips before deploying ADF JSON files List <AdfFileInfo> pipelines = validFiles.Where(x => x.FileType == FileType.Pipeline).ToList(); List <CustomActivityPackageInfo> packages = pipelines.SelectMany(x => x.CustomActivityPackages).Distinct().ToList(); if (packages.Any()) { result &= await DeployCustomActivities(packages, validFiles, outputFolder); } List <AdfFileInfo> linkedServices = validFiles.Where(x => x.FileType == FileType.LinkedService).ToList(); if (linkedServices.Any()) { logger.Write(string.Empty); logger.Write("Deploying LinkedServices", "Black"); // Deploy non batch linked services first var linkedServiceTaskDict = new Dictionary <string, Task <LinkedServiceCreateOrUpdateResponse> >(); foreach (var linkedService in linkedServices.Where(x => !x.SubType.Equals("AzureBatch", StringComparison.InvariantCultureIgnoreCase))) { linkedServiceTaskDict.Add(linkedService.Name, client.LinkedServices.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, dataFactoryName, linkedService.Name, new LinkedServiceCreateOrUpdateWithRawJsonContentParameters(linkedService.FileContents))); } foreach (var item in linkedServiceTaskDict) { try { LinkedServiceCreateOrUpdateResponse response = await item.Value; if (response.StatusCode == HttpStatusCode.OK) { logger.Write($"Linked service '{response.LinkedService.Name}' uploaded successfully.", "Green"); } else { logger.Write($"Linked service '{response.LinkedService.Name}' did not upload successfully. Response status: {response.Status}", "Red"); result = false; } } catch (Exception e) { logger.Write($"Linked service '{item.Key}' did not upload successfully. Error: {e.Message}", "Red"); logger.WriteError(e); result = false; } } // Deploy batch linked services next var batchLinkedServiceTaskDict = new Dictionary <string, Task <LinkedServiceCreateOrUpdateResponse> >(); foreach (var batchLinkedService in linkedServices.Where(x => x.SubType.Equals("AzureBatch", StringComparison.InvariantCultureIgnoreCase))) { batchLinkedServiceTaskDict.Add(batchLinkedService.Name, client.LinkedServices.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, dataFactoryName, batchLinkedService.Name, new LinkedServiceCreateOrUpdateWithRawJsonContentParameters(batchLinkedService.FileContents))); } foreach (var item in batchLinkedServiceTaskDict) { try { LinkedServiceCreateOrUpdateResponse response = await item.Value; if (response.StatusCode == HttpStatusCode.OK) { logger.Write($"Linked service '{response.LinkedService.Name}' uploaded successfully.", "Green"); } else { logger.Write($"Linked service '{response.LinkedService.Name}' did not upload successfully. Response status: {response.Status}", "Red"); result = false; } } catch (Exception e) { logger.Write($"Linked service '{item.Key}' did not upload successfully. Error: {e.Message}", "Red"); logger.WriteError(e); result = false; } } } List <AdfFileInfo> tables = validFiles.Where(x => x.FileType == FileType.Table).ToList(); if (tables.Any()) { logger.Write(string.Empty); logger.Write("Deploying tables", "Black"); // Deploy tables next var tableTaskDict = new Dictionary <string, Task <DatasetCreateOrUpdateResponse> >(); foreach (AdfFileInfo adfJsonFile in tables) { try { Task <DatasetCreateOrUpdateResponse> tableTask = client.Datasets.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, dataFactoryName, adfJsonFile.Name, new DatasetCreateOrUpdateWithRawJsonContentParameters(adfJsonFile.FileContents)); tableTaskDict.Add(adfJsonFile.Name, tableTask); } catch (Exception e) { logger.Write($"An error occurred uploading table '{adfJsonFile.Name}': " + e.Message, "Red"); logger.WriteError(e); result = false; } } foreach (var task in tableTaskDict) { try { DatasetCreateOrUpdateResponse response = await task.Value; if (response.StatusCode == HttpStatusCode.OK) { logger.Write($"Table '{task.Key}' uploaded successfully.", "Green"); } else { logger.Write( $"Table '{task.Key}' did not upload successfully. Response status: {response.Status}", "Red"); result = false; } } catch (CloudException ex) { if (ex.Error.Code == "TableAvailabilityUpdateNotSupported") { logger.Write($"It looks like you are trying to change the availability for the Table '{task.Key}'. Currently this is not supported by ADF so as work around you should delete the dataset and related pipleline in the Data Factory '{dataFactoryName}' and run the publish again.", "Red"); } else { logger.Write($"Table '{task.Key}' did not upload successfully. An error occurred: " + ex.Message, "Red"); } logger.WriteError(ex); result = false; } catch (Exception ex) { logger.WriteError(ex); logger.Write($"Table '{task.Key}' did not upload successfully. An error occurred: " + ex.Message, "Red"); result = false; } } } if (pipelines.Any()) { logger.Write(string.Empty); logger.Write("Deploying pipelines", "Black"); // Deploy pipelines last var pipelineTaskDict = new Dictionary <string, Task <PipelineCreateOrUpdateResponse> >(); foreach (AdfFileInfo adfJsonFile in pipelines) { pipelineTaskDict.Add(adfJsonFile.Name, client.Pipelines.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, dataFactoryName, adfJsonFile.Name, new PipelineCreateOrUpdateWithRawJsonContentParameters(adfJsonFile.FileContents))); } foreach (var item in pipelineTaskDict) { try { PipelineCreateOrUpdateResponse response = await item.Value; if (response.StatusCode == HttpStatusCode.OK) { logger.Write($"Pipeline '{response.Pipeline.Name}' uploaded successfully.", "Green"); } else { logger.Write($"Pipeline '{response.Pipeline.Name}' did not upload successfully. Response status: {response.Status}", "Red"); result = false; } } catch (Exception e) { logger.WriteError(e); logger.Write($"An error occurred uploading pipeline '{item.Key}': " + e.Message, "Red"); result = false; } } } return(result); }