private static Dictionary <string, DeployConfigInfo> deployConfigs; //服务器列表
public static Dictionary <string, DeployConfigInfo> GetDeployConfigInfos()
{
if (deployConfigs == null)
{
string jsonstr = "";// SingletonMB<ResourceManagerController>.GetInstance().LoadData(AssetConst.JsonData, "deploy");
JsonData jd = JsonMapper.ToObject(jsonstr);
JsonData items = jd["items"];
Dictionary <string, DeployConfigInfo> cinfos = new Dictionary <string, DeployConfigInfo>();
int count = items.Count;
for (int i = 0; i < count; i++)
{
JsonData item = items[i];
string json = JsonMapper.ToJson(item);
DeployConfigInfo cinfo = GetConfigInfo("deploy", json) as DeployConfigInfo;
if (cinfo != null)
{
if (!cinfos.ContainsKey(cinfo.param))
{
cinfos.Add(cinfo.param, cinfo);
}
}
}
deployConfigs = cinfos;
}
return(deployConfigs);
}
/// <summary>
/// Publishes to the specified data factory given a prebuilt output folder
/// </summary>
/// <returns>True if the publish succeeds othewise false</returns>
public async Task <bool> PublishFromOutputFolder(string adfOutputFolder, string dataFactoryResourceGroup, string dataFactoryName)
{
try
{
// Get schemas used for determining ADF files
await adfFileHelper.GetSchemas(httpClient);
List <string> filesToProcess = Directory.GetFiles(adfOutputFolder, "*.json", SearchOption.TopDirectoryOnly).ToList();
var lowerExcludeList = settingsContext.FilesToExclude == null || !settingsContext.FilesToExclude.Any()
? new List <string>()
: settingsContext.FilesToExclude.Select(x => x.ToLowerInvariant());
filesToProcess = filesToProcess.Where(x =>
{
var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(x);
return(fileNameWithoutExtension != null &&
!lowerExcludeList.Contains(fileNameWithoutExtension.ToLowerInvariant()));
}).ToList();
string deployConfigPath = string.IsNullOrEmpty(settingsContext.DeploymentConfigName)
? null
: filesToProcess.FirstOrDefault(x =>
{
var fileName = Path.GetFileName(x);
return(fileName != null &&
fileName.Equals(Path.GetFileNameWithoutExtension(settingsContext.DeploymentConfigName) + ".json",
StringComparison.InvariantCultureIgnoreCase));
});
DeployConfigInfo deployConfig = null;
logger.Write(string.Empty);
if (!string.IsNullOrEmpty(deployConfigPath))
{
logger.Write("Using deployment configuration file: " + deployConfigPath);
deployConfig = adfFileHelper.GetDeployConfigInfo(deployConfigPath);
}
else
{
logger.Write("No deployment configuration file found.", "Orange");
}
AdfDeploy adf = new AdfDeploy(adfFileHelper, logger, blob, settingsContext, dataFactoryResourceGroup, dataFactoryName);
return(await adf.Deploy(filesToProcess, adfOutputFolder, deployConfig));
}
catch (Exception e)
{
logger.Write($"Error: {e.Message}");
logger.WriteError(e);
return(false);
}
}
/// <summary>
/// Resolves the deployment settings from the deployment config.
/// </summary>
private JObject ResolveDeploymentSettings(JObject jObject, string name, DeployConfigInfo deployConfig)
{
// Update with values from delpoyment config if exists
if (deployConfig.DeploymentDictionary.Count > 0 && deployConfig.DeploymentDictionary.ContainsKey(name))
{
foreach (KeyValuePair <string, string> pair in deployConfig.DeploymentDictionary[name])
{
JToken token = jObject.SelectToken(pair.Key);
if (token == null)
{
logger.Write(
$"The deployment configuration file '{deployConfig.FileName}' contains a substitution for '{name}' however the JPath '{pair.Key}' is not found in '{name}'.");
}
else
{
token.Replace(pair.Value);
}
}
}
return(jObject);
}
/// <summary>
/// Gets the information on the ADF file.
/// </summary>
/// <param name="filePath">The file path.</param>
/// <param name="deployConfig">The deploy configuration.</param>
public async Task <AdfFileInfo> GetFileInfo(string filePath, DeployConfigInfo deployConfig = null)
{
string fileContents = File.ReadAllText(filePath);
string fileName = Path.GetFileNameWithoutExtension(filePath);
// Initialize props to default values
AdfFileInfo fileInfo = new AdfFileInfo
{
FileType = FileType.Unknown,
SubType = string.Empty,
Name = string.Empty,
IsValid = false,
FileContents = fileContents,
FileName = fileName
};
JObject jObject = null;
try
{
jObject = JObject.Parse(fileContents);
JObject propertyJObject = jObject.GetValue("properties", StringComparison.OrdinalIgnoreCase) as JObject;
JToken nameJToken = jObject.GetValue("name", StringComparison.OrdinalIgnoreCase);
if (propertyJObject == null || nameJToken == null)
{
logger.Write($"{fileInfo.FileName} is a not a valid ADF file.");
return(fileInfo);
}
fileInfo.Name = nameJToken.ToObject <string>();
if (deployConfig != null)
{
// Update ADF files with deploymnet settings if they exist
jObject = ResolveDeploymentSettings(jObject, fileInfo.Name, deployConfig);
fileInfo.FileContents = jObject.ToString();
}
JToken typeJToken = propertyJObject.GetValue("type", StringComparison.OrdinalIgnoreCase);
if (typeJToken != null)
{
fileInfo.SubType = typeJToken.ToObject <string>();
}
if (fileInfo.SubType == "CPSServiceBusProxy" || jObject.IsValid(jsonLinkedServiceSchema))
{
fileInfo.FileType = FileType.LinkedService;
fileInfo.IsValid = true;
logger.Write($"Retreived LinkedService: {fileInfo.FileName}");
}
else if (jObject.IsValid(jsonTableSchema))
{
fileInfo.FileType = FileType.Table;
fileInfo.IsValid = true;
logger.Write($"Retreived Dataset: {fileInfo.FileName}");
}
else if (jObject.IsValid(jsonPipelineSchema))
{
fileInfo.FileType = FileType.Pipeline;
fileInfo.IsValid = true;
logger.Write($"Retreived Pipeline: {fileInfo.FileName}");
// Get all custom activity packages if available
JArray activities = propertyJObject.GetValue("activities", StringComparison.InvariantCultureIgnoreCase) as JArray;
if (activities != null)
{
fileInfo.CustomActivityPackages = new List <CustomActivityPackageInfo>();
foreach (JObject activity in activities)
{
JToken activityTypeJToken = activity.GetValue("type", StringComparison.OrdinalIgnoreCase);
if (activityTypeJToken != null && activityTypeJToken.ToObject <string>().Equals("DotNetActivity", StringComparison.CurrentCultureIgnoreCase))
{
CustomActivityPackageInfo packageInfo = new CustomActivityPackageInfo();
packageInfo.PackageLinkedService = activity.SelectToken("$.typeProperties.packageLinkedService")?.ToObject <string>();
packageInfo.PackageFile = activity.SelectToken("$.typeProperties.packageFile")?.ToObject <string>();
logger.Write($"Retreived Custom Activity package: {packageInfo.PackageFile}");
fileInfo.CustomActivityPackages.Add(packageInfo);
}
}
}
}
else
{
fileInfo.FileType = FileType.Unknown;
logger.Write($"{fileInfo.FileName} is a not a valid ADF file.");
}
}
catch (Exception e)
{
fileInfo.ErrorException = e;
logger.Write($"{fileInfo.FileName} is a not a valid ADF file. Error message: {e}");
logger.WriteError(e);
}
if (fileInfo.IsValid)
{
// Search for keyvault tokens and resolve them
string keyVaultResolvedContents = await ResolveKeyVault(fileInfo.FileContents);
if (fileInfo.FileContents != keyVaultResolvedContents)
{
fileInfo.FileContents = keyVaultResolvedContents;
fileInfo.JObject = JObject.Parse(fileInfo.FileContents);
}
else
{
fileInfo.JObject = jObject;
}
}
return(fileInfo);
}
/// <summary>
/// Deploys the specified ADF files and custom activity packages to Azure.
/// </summary>
/// <param name="filesToProcess">The files to process.</param>
/// <param name="outputFolder">The output folder which contains the files and custom activity zips.</param>
/// <param name="deployConfig">The deployment configuration information.</param>
public async Task <bool> Deploy(List <string> filesToProcess, string outputFolder, DeployConfigInfo deployConfig = null)
{
bool result = true;
logger.Write(string.Empty);
logger.Write($"Getting all ADF resources to deploy to Azure from output folder '{outputFolder}'", "Black");
List <Task <AdfFileInfo> > allFilesTasks = filesToProcess.Select(async x => await adfFileHelper.GetFileInfo(x, deployConfig)).ToList();
List <AdfFileInfo> allFiles = new List <AdfFileInfo>();
foreach (var allFilesTask in allFilesTasks)
{
allFiles.Add(await allFilesTask);
}
List <AdfFileInfo> validFiles = allFiles.Where(x => x.IsValid).ToList();
if (!validFiles.Any())
{
logger.Write($"No valid ADF files found in '{outputFolder}'", "Red");
return(false);
}
logger.Write($"{validFiles.Count} file{(validFiles.Count == 1 ? string.Empty : "s")} retreived");
logger.Write(string.Empty);
logger.Write($"Begin deploying ADF resources to '{dataFactoryName}'", "Black");
// Log invalid files
List <AdfFileInfo> invalidFiles = allFiles.Where(x => !x.IsValid).ToList();
if (invalidFiles.Any())
{
logger.Write("The following files found in the output folder will not be published:");
foreach (AdfFileInfo invalidFile in invalidFiles)
{
logger.Write(invalidFile.FileName);
}
}
DataFactoryManagementClient client = GetDataFactoryManagementClient();
// Deploy Package Zips before deploying ADF JSON files
List <AdfFileInfo> pipelines = validFiles.Where(x => x.FileType == FileType.Pipeline).ToList();
List <CustomActivityPackageInfo> packages = pipelines.SelectMany(x => x.CustomActivityPackages).Distinct().ToList();
if (packages.Any())
{
result &= await DeployCustomActivities(packages, validFiles, outputFolder);
}
List <AdfFileInfo> linkedServices = validFiles.Where(x => x.FileType == FileType.LinkedService).ToList();
if (linkedServices.Any())
{
logger.Write(string.Empty);
logger.Write("Deploying LinkedServices", "Black");
// Deploy non batch linked services first
var linkedServiceTaskDict = new Dictionary <string, Task <LinkedServiceCreateOrUpdateResponse> >();
foreach (var linkedService in linkedServices.Where(x => !x.SubType.Equals("AzureBatch", StringComparison.InvariantCultureIgnoreCase)))
{
linkedServiceTaskDict.Add(linkedService.Name,
client.LinkedServices.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName,
dataFactoryName, linkedService.Name,
new LinkedServiceCreateOrUpdateWithRawJsonContentParameters(linkedService.FileContents)));
}
foreach (var item in linkedServiceTaskDict)
{
try
{
LinkedServiceCreateOrUpdateResponse response = await item.Value;
if (response.StatusCode == HttpStatusCode.OK)
{
logger.Write($"Linked service '{response.LinkedService.Name}' uploaded successfully.", "Green");
}
else
{
logger.Write($"Linked service '{response.LinkedService.Name}' did not upload successfully. Response status: {response.Status}", "Red");
result = false;
}
}
catch (Exception e)
{
logger.Write($"Linked service '{item.Key}' did not upload successfully. Error: {e.Message}", "Red");
logger.WriteError(e);
result = false;
}
}
// Deploy batch linked services next
var batchLinkedServiceTaskDict = new Dictionary <string, Task <LinkedServiceCreateOrUpdateResponse> >();
foreach (var batchLinkedService in linkedServices.Where(x => x.SubType.Equals("AzureBatch", StringComparison.InvariantCultureIgnoreCase)))
{
batchLinkedServiceTaskDict.Add(batchLinkedService.Name,
client.LinkedServices.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName,
dataFactoryName, batchLinkedService.Name,
new LinkedServiceCreateOrUpdateWithRawJsonContentParameters(batchLinkedService.FileContents)));
}
foreach (var item in batchLinkedServiceTaskDict)
{
try
{
LinkedServiceCreateOrUpdateResponse response = await item.Value;
if (response.StatusCode == HttpStatusCode.OK)
{
logger.Write($"Linked service '{response.LinkedService.Name}' uploaded successfully.", "Green");
}
else
{
logger.Write($"Linked service '{response.LinkedService.Name}' did not upload successfully. Response status: {response.Status}", "Red");
result = false;
}
}
catch (Exception e)
{
logger.Write($"Linked service '{item.Key}' did not upload successfully. Error: {e.Message}", "Red");
logger.WriteError(e);
result = false;
}
}
}
List <AdfFileInfo> tables = validFiles.Where(x => x.FileType == FileType.Table).ToList();
if (tables.Any())
{
logger.Write(string.Empty);
logger.Write("Deploying tables", "Black");
// Deploy tables next
var tableTaskDict = new Dictionary <string, Task <DatasetCreateOrUpdateResponse> >();
foreach (AdfFileInfo adfJsonFile in tables)
{
try
{
Task <DatasetCreateOrUpdateResponse> tableTask = client.Datasets.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, dataFactoryName,
adfJsonFile.Name, new DatasetCreateOrUpdateWithRawJsonContentParameters(adfJsonFile.FileContents));
tableTaskDict.Add(adfJsonFile.Name, tableTask);
}
catch (Exception e)
{
logger.Write($"An error occurred uploading table '{adfJsonFile.Name}': " + e.Message, "Red");
logger.WriteError(e);
result = false;
}
}
foreach (var task in tableTaskDict)
{
try
{
DatasetCreateOrUpdateResponse response = await task.Value;
if (response.StatusCode == HttpStatusCode.OK)
{
logger.Write($"Table '{task.Key}' uploaded successfully.", "Green");
}
else
{
logger.Write(
$"Table '{task.Key}' did not upload successfully. Response status: {response.Status}",
"Red");
result = false;
}
}
catch (CloudException ex)
{
if (ex.Error.Code == "TableAvailabilityUpdateNotSupported")
{
logger.Write($"It looks like you are trying to change the availability for the Table '{task.Key}'. Currently this is not supported by ADF so as work around you should delete the dataset and related pipleline in the Data Factory '{dataFactoryName}' and run the publish again.", "Red");
}
else
{
logger.Write($"Table '{task.Key}' did not upload successfully. An error occurred: " + ex.Message, "Red");
}
logger.WriteError(ex);
result = false;
}
catch (Exception ex)
{
logger.WriteError(ex);
logger.Write($"Table '{task.Key}' did not upload successfully. An error occurred: " + ex.Message, "Red");
result = false;
}
}
}
if (pipelines.Any())
{
logger.Write(string.Empty);
logger.Write("Deploying pipelines", "Black");
// Deploy pipelines last
var pipelineTaskDict = new Dictionary <string, Task <PipelineCreateOrUpdateResponse> >();
foreach (AdfFileInfo adfJsonFile in pipelines)
{
pipelineTaskDict.Add(adfJsonFile.Name, client.Pipelines.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, dataFactoryName,
adfJsonFile.Name, new PipelineCreateOrUpdateWithRawJsonContentParameters(adfJsonFile.FileContents)));
}
foreach (var item in pipelineTaskDict)
{
try
{
PipelineCreateOrUpdateResponse response = await item.Value;
if (response.StatusCode == HttpStatusCode.OK)
{
logger.Write($"Pipeline '{response.Pipeline.Name}' uploaded successfully.", "Green");
}
else
{
logger.Write($"Pipeline '{response.Pipeline.Name}' did not upload successfully. Response status: {response.Status}", "Red");
result = false;
}
}
catch (Exception e)
{
logger.WriteError(e);
logger.Write($"An error occurred uploading pipeline '{item.Key}': " + e.Message, "Red");
result = false;
}
}
}
return(result);
}
