public AzurePipelines.Job[] ProcessJobFromPipelineRootV2(string poolYaml, string strategyYaml, string stepsYaml) { Pool pool = null; if (poolYaml != null) { GeneralProcessing gp = new GeneralProcessing(_verbose); pool = gp.ProcessPoolV2(poolYaml); } AzurePipelines.Strategy strategy = null; try { //Most often, the pool will be in this structure strategy = GenericObjectSerialization.DeserializeYaml <AzurePipelines.Strategy>(strategyYaml); } catch (Exception ex) { ConversionUtility.WriteLine($"DeserializeYaml<AzurePipelines.Strategy>(strategyYaml) swallowed an exception: " + ex.Message, _verbose); } AzurePipelines.Step[] steps = null; if (stepsYaml != null) { try { steps = GenericObjectSerialization.DeserializeYaml <AzurePipelines.Step[]>(stepsYaml); } catch (Exception ex) { ConversionUtility.WriteLine($"DeserializeYaml<AzurePipelines.Step[]>(stepsYaml) swallowed an exception: " + ex.Message, _verbose); } } AzurePipelines.Job job = new AzurePipelines.Job { pool = pool, strategy = strategy, steps = steps }; //Don't add the build name unless there is content if (job.pool != null || job.strategy != null || steps != null) { AzurePipelines.Job[] jobs = new AzurePipelines.Job[1]; job.job = "build"; jobs[0] = job; return(jobs); } else { return(null); } }
public GitHubActions.Job ProcessJob(AzurePipelines.Job job, AzurePipelines.Resources resources) { GeneralProcessing generalProcessing = new GeneralProcessing(_verbose); VariablesProcessing vp = new VariablesProcessing(_verbose); StepsProcessing sp = new StepsProcessing(); GitHubActions.Job newJob = new GitHubActions.Job { name = job.displayName, needs = job.dependsOn, _if = ConditionsProcessing.TranslateConditions(job.condition), runs_on = generalProcessing.ProcessPool(job.pool), strategy = generalProcessing.ProcessStrategy(job.strategy), container = generalProcessing.ProcessContainer(resources), env = vp.ProcessSimpleVariables(job.variables), timeout_minutes = job.timeoutInMinutes, steps = sp.AddSupportingSteps(job.steps) }; MatrixVariableName = generalProcessing.MatrixVariableName; VariableList = vp.VariableList; if (newJob.steps == null & job.template != null) { //Initialize the array with no items job.steps = new AzurePipelines.Step[0]; //Process the steps, adding the default checkout step newJob.steps = sp.AddSupportingSteps(job.steps, true); //TODO: There is currently no conversion path for templates newJob.job_message += "Note: Azure DevOps template does not have an equivalent in GitHub Actions yet"; } else if (newJob.steps == null && job.strategy?.runOnce?.deploy?.steps != null) { //Initialize the array with no items job.steps = new AzurePipelines.Step[0]; //Process the steps, adding the default checkout step newJob.steps = sp.AddSupportingSteps(job.strategy?.runOnce?.deploy?.steps, false); //TODO: There is currently no conversion path for templates newJob.job_message += "Note: Azure DevOps strategy>runOnce>deploy does not have an equivalent in GitHub Actions yet"; } if (job.environment != null) { newJob.job_message += "Note: Azure DevOps job environment does not have an equivalent in GitHub Actions yet"; } if (job.continueOnError == true) { newJob.continue_on_error = job.continueOnError; } return(newJob); }
public AzurePipelines.Job[] ExtractAzurePipelinesJobsV2(JToken jobsJson, string strategyYaml) { GeneralProcessing gp = new GeneralProcessing(_verbose); AzurePipelines.Job[] jobs = new AzurePipelines.Job[jobsJson.Count()]; if (jobsJson != null) { int i = 0; foreach (JToken jobJson in jobsJson) { AzurePipelines.Job job = new AzurePipelines.Job { job = jobJson["job"]?.ToString(), deployment = jobJson["deployment"]?.ToString(), displayName = jobJson["displayName"]?.ToString(), template = jobJson["template"]?.ToString() }; if (jobJson["pool"] != null) { job.pool = gp.ProcessPoolV2(jobJson["pool"].ToString()); } if (jobJson["strategy"] != null) { job.strategy = gp.ProcessStrategyV2(jobJson["strategy"].ToString()); } else if (strategyYaml != null) { job.strategy = gp.ProcessStrategyV2(strategyYaml); } if (jobJson["dependsOn"] != null) { job.dependsOn = gp.ProcessDependsOnV2(jobJson["dependsOn"].ToString()); } if (jobJson["condition"] != null) { job.condition = ConditionsProcessing.TranslateConditions(jobJson["condition"].ToString()); } if (jobJson["environment"] != null) { job.environment = gp.ProcessEnvironmentV2(jobJson["environment"].ToString()); } if (jobJson["timeoutInMinutes"] != null) { int.TryParse(jobJson["timeoutInMinutes"].ToString(), out int timeOut); if (timeOut > 0) { job.timeoutInMinutes = timeOut; } } if (jobJson["continueOnError"] != null) { bool.TryParse(jobJson["continueOnError"].ToString(), out bool continueOnError); job.continueOnError = continueOnError; } if (jobJson["variables"] != null) { VariablesProcessing vp = new VariablesProcessing(_verbose); job.variables = vp.ProcessParametersAndVariablesV2(null, jobJson["variables"].ToString()); } if (jobJson["steps"] != null) { try { job.steps = GenericObjectSerialization.DeserializeYaml <AzurePipelines.Step[]>(jobJson["steps"].ToString()); } catch (Exception ex) { ConversionUtility.WriteLine($"DeserializeYaml<AzurePipelines.Step[]>(jobJson[\"steps\"].ToString() swallowed an exception: " + ex.Message, _verbose); } } jobs[i] = job; i++; } } return(jobs); }
/// <summary> /// Process an Azure DevOps Pipeline, converting it to a GitHub Action /// </summary> /// <param name="azurePipeline">Azure DevOps Pipeline object</param> /// <param name="simpleTrigger">When the YAML has a simple trigger, (String[]). Can be null</param> /// <param name="complexTrigger">When the YAML has a complex trigger. Can be null</param> /// <returns>GitHub Actions object</returns> public GitHubActionsRoot ProcessPipeline(AzurePipelinesRoot <TTriggers, TVariables> azurePipeline, string[] simpleTrigger, AzurePipelines.Trigger complexTrigger, Dictionary <string, string> simpleVariables, AzurePipelines.Variable[] complexVariables) { VariableList = new List <string>(); GeneralProcessing generalProcessing = new GeneralProcessing(_verbose); GitHubActionsRoot gitHubActions = new GitHubActionsRoot(); //Name if (azurePipeline.name != null) { gitHubActions.name = azurePipeline.name; } //Container if (azurePipeline.container != null) { gitHubActions.messages.Add("TODO: Container conversion not yet done, we need help!: https://github.com/samsmithnz/AzurePipelinesToGitHubActionsConverter/issues/39"); } //Triggers for pushs TriggerProcessing tp = new TriggerProcessing(_verbose); if (azurePipeline.trigger != null) { if (complexTrigger != null) { gitHubActions.on = tp.ProcessComplexTrigger(complexTrigger); } else if (simpleTrigger != null) { gitHubActions.on = tp.ProcessSimpleTrigger(simpleTrigger); } } //Triggers for pull requests if (azurePipeline.pr != null) { GitHubActions.Trigger pr = tp.ProcessPullRequest(azurePipeline.pr); if (gitHubActions.on == null) { gitHubActions.on = pr; } else { gitHubActions.on.pull_request = pr.pull_request; } } //pool/demands if (azurePipeline.pool != null && azurePipeline.pool.demands != null) { gitHubActions.messages.Add("Note: GitHub Actions does not have a 'demands' command on 'runs-on' yet"); } //schedules if (azurePipeline.schedules != null) { string[] schedules = tp.ProcessSchedules(azurePipeline.schedules); if (gitHubActions.on == null) { gitHubActions.on = new GitHubActions.Trigger(); } gitHubActions.on.schedule = schedules; } //Resources if (azurePipeline.resources != null) { //Note: Containers is in the jobs - this note should be removed once pipeliens and repositories is moved too //TODO: There is currently no conversion path for pipelines if (azurePipeline.resources.pipelines != null) { gitHubActions.messages.Add("TODO: Resource pipelines conversion not yet done: https://github.com/samsmithnz/AzurePipelinesToGitHubActionsConverter/issues/8"); if (azurePipeline.resources.pipelines.Length > 0) { if (azurePipeline.resources.pipelines[0].pipeline != null) { ConversionUtility.WriteLine("pipeline: " + azurePipeline.resources.pipelines[0].pipeline, _verbose); } if (azurePipeline.resources.pipelines[0].project != null) { ConversionUtility.WriteLine("project: " + azurePipeline.resources.pipelines[0].project, _verbose); } if (azurePipeline.resources.pipelines[0].source != null) { ConversionUtility.WriteLine("source: " + azurePipeline.resources.pipelines[0].source, _verbose); } if (azurePipeline.resources.pipelines[0].branch != null) { ConversionUtility.WriteLine("branch: " + azurePipeline.resources.pipelines[0].branch, _verbose); } if (azurePipeline.resources.pipelines[0].version != null) { ConversionUtility.WriteLine("version: " + azurePipeline.resources.pipelines[0].version, _verbose); } if (azurePipeline.resources.pipelines[0].trigger != null) { if (azurePipeline.resources.pipelines[0].trigger.autoCancel) { ConversionUtility.WriteLine("autoCancel: " + azurePipeline.resources.pipelines[0].trigger.autoCancel, _verbose); } if (azurePipeline.resources.pipelines[0].trigger.batch) { ConversionUtility.WriteLine("batch: " + azurePipeline.resources.pipelines[0].trigger.batch, _verbose); } } } } //TODO: There is currently no conversion path for repositories if (azurePipeline.resources.repositories != null) { gitHubActions.messages.Add("TODO: Resource repositories conversion not yet done: https://github.com/samsmithnz/AzurePipelinesToGitHubActionsConverter/issues/8"); if (azurePipeline.resources.repositories.Length > 0) { if (azurePipeline.resources.repositories[0].repository != null) { ConversionUtility.WriteLine("repository: " + azurePipeline.resources.repositories[0].repository, _verbose); } if (azurePipeline.resources.repositories[0].type != null) { ConversionUtility.WriteLine("type: " + azurePipeline.resources.repositories[0].type, _verbose); } if (azurePipeline.resources.repositories[0].name != null) { ConversionUtility.WriteLine("name: " + azurePipeline.resources.repositories[0].name, _verbose); } if (azurePipeline.resources.repositories[0]._ref != null) { ConversionUtility.WriteLine("ref: " + azurePipeline.resources.repositories[0]._ref, _verbose); } if (azurePipeline.resources.repositories[0].endpoint != null) { ConversionUtility.WriteLine("endpoint: " + azurePipeline.resources.repositories[0].endpoint, _verbose); } if (azurePipeline.resources.repositories[0].connection != null) { ConversionUtility.WriteLine("connection: " + azurePipeline.resources.repositories[0].connection, _verbose); } if (azurePipeline.resources.repositories[0].source != null) { ConversionUtility.WriteLine("source: " + azurePipeline.resources.repositories[0].source, _verbose); } } } } //Stages (Note: stages are not yet present in actions, we are merging them into one giant list of jobs, appending the stage name to jobs to keep names unique) if (azurePipeline.stages != null) { //Count the number of jobs and initialize the jobs array with that number int jobCounter = 0; foreach (Stage stage in azurePipeline.stages) { if (stage.jobs != null) { jobCounter += stage.jobs.Length; } } azurePipeline.jobs = new AzurePipelines.Job[jobCounter]; //We are going to take each stage and assign it a set of jobs int currentIndex = 0; foreach (Stage stage in azurePipeline.stages) { if (stage.jobs != null) { int j = 0; for (int i = 0; i < stage.jobs.Length; i++) { //Get the job name string jobName = ConversionUtility.GenerateJobName(stage.jobs[i], currentIndex); //Rename the job, using the stage name as prefix, so that we keep the job names unique stage.jobs[j].job = stage.stage + "_Stage_" + jobName; ConversionUtility.WriteLine("This variable is not needed in actions: " + stage.displayName, _verbose); azurePipeline.jobs[currentIndex] = stage.jobs[j]; azurePipeline.jobs[currentIndex].condition = stage.condition; //Move over the variables, the stage variables will need to be applied to each job if (stage.variables != null && stage.variables.Count > 0) { azurePipeline.jobs[currentIndex].variables = new Dictionary <string, string>(); foreach (KeyValuePair <string, string> stageVariable in stage.variables) { azurePipeline.jobs[currentIndex].variables.Add(stageVariable.Key, stageVariable.Value); } } j++; currentIndex++; } } } } //Jobs (when no stages are defined) if (azurePipeline.jobs != null) { //If there is a parent strategy, and no child strategy, load in the parent //This is not perfect... if (azurePipeline.strategy != null) { foreach (AzurePipelines.Job item in azurePipeline.jobs) { if (item.strategy == null) { item.strategy = azurePipeline.strategy; } } } gitHubActions.jobs = ProcessJobs(azurePipeline.jobs, azurePipeline.resources); if (gitHubActions.jobs.Count == 0) { gitHubActions.messages.Add("Note that although having no jobs is valid YAML, it is not a valid GitHub Action."); } } //Pool + Steps (When there are no jobs defined) if ((azurePipeline.pool != null && azurePipeline.jobs == null) || (azurePipeline.steps != null && azurePipeline.steps.Length > 0)) { //Steps only have one job, so we just create it here StepsProcessing sp = new StepsProcessing(); gitHubActions.jobs = new Dictionary <string, GitHubActions.Job> { { "build", new GitHubActions.Job { runs_on = generalProcessing.ProcessPool(azurePipeline.pool), strategy = generalProcessing.ProcessStrategy(azurePipeline.strategy), container = generalProcessing.ProcessContainer(azurePipeline.resources), //resources = ProcessResources(azurePipeline.resources), steps = sp.AddSupportingSteps(azurePipeline.steps) } } }; MatrixVariableName = generalProcessing.MatrixVariableName; } //Variables VariablesProcessing vp = new VariablesProcessing(_verbose); if (azurePipeline.variables != null) { if (complexVariables != null) { gitHubActions.env = vp.ProcessComplexVariables(complexVariables); VariableList.AddRange(vp.VariableList); } else if (simpleVariables != null) { gitHubActions.env = vp.ProcessSimpleVariables(simpleVariables); VariableList.AddRange(vp.VariableList); } } else if (azurePipeline.parameters != null) { //For now, convert the parameters to variables gitHubActions.env = vp.ProcessSimpleVariables(azurePipeline.parameters); } return(gitHubActions); }
public Dictionary <string, GitHubActions.Job> ProcessStagesV2(JToken stagesJson, string strategyYaml) { AzurePipelines.Job[] jobs = null; List <AzurePipelines.Stage> stages = new List <AzurePipelines.Stage>(); if (stagesJson != null) { //for each stage foreach (JToken stageJson in stagesJson) { AzurePipelines.Stage stage = new AzurePipelines.Stage { stage = stageJson["stage"]?.ToString(), displayName = stageJson["displayName"]?.ToString(), condition = stageJson["condition"]?.ToString() }; if (stageJson["dependsOn"] != null) { GeneralProcessing gp = new GeneralProcessing(_verbose); stage.dependsOn = gp.ProcessDependsOnV2(stageJson["dependsOn"].ToString()); } if (stageJson["variables"] != null) { VariablesProcessing vp = new VariablesProcessing(_verbose); stage.variables = vp.ProcessParametersAndVariablesV2(null, stageJson["variables"].ToString()); } if (stageJson["jobs"] != null) { JobProcessing jp = new JobProcessing(_verbose); stage.jobs = jp.ExtractAzurePipelinesJobsV2(stageJson["jobs"], strategyYaml); } stages.Add(stage); } //process the jobs if (stages != null) { int jobCount = 0; foreach (Stage stage in stages) { if (stage.jobs != null) { jobCount += stage.jobs.Length; } } jobs = new AzurePipelines.Job[jobCount]; //Giant nested loop ahead. Loop through stages, looking for all jobs int jobIndex = 0; foreach (Stage stage in stages) { if (stage.jobs != null) { for (int i = 0; i < stage.jobs.Length; i++) { jobs[jobIndex] = stage.jobs[i]; if (stage.variables != null) { if (jobs[jobIndex].variables == null) { jobs[jobIndex].variables = new Dictionary <string, string>(); } foreach (KeyValuePair <string, string> stageVariable in stage.variables) { //Add the stage variable if it doesn't already exist if (jobs[jobIndex].variables.ContainsKey(stageVariable.Key) == false) { jobs[jobIndex].variables.Add(stageVariable.Key, stageVariable.Value); } } } if (stage.condition != null) { jobs[jobIndex].condition = stage.condition; } //Get the job name string jobName = ConversionUtility.GenerateJobName(stage.jobs[i], jobIndex); //Rename the job, using the stage name as prefix, so that we keep the job names unique jobs[jobIndex].job = stage.stage + "_Stage_" + jobName; jobIndex++; } } } } } //Build the final list of GitHub jobs and return it if (jobs != null) { Dictionary <string, GitHubActions.Job> gitHubJobs = new Dictionary <string, GitHubActions.Job>(); foreach (AzurePipelines.Job job in jobs) { JobProcessing jobProcessing = new JobProcessing(_verbose); gitHubJobs.Add(job.job, jobProcessing.ProcessJob(job, null)); } return(gitHubJobs); } else { return(null); } }
/// <summary> /// V2 plan: /// 1. get the yaml /// 2. converting the yaml into a json document /// 3. parse the json document into azure pipelines sub-objects /// 4. put it together into one azure pipelines object /// 5. convert the azure pipelines object to github action /// </summary> /// <param name="yaml"></param> /// <returns></returns> private ConversionResponse ConvertAzurePipelineToGitHubActionV2(string yaml) { string gitHubYaml = ""; List <string> variableList = new List <string>(); List <string> stepComments = new List <string>(); //convert the yaml into json, it's easier to parse JObject json = null; if (yaml != null) { //Clean up the YAML to remove conditional insert statements string processedYaml = ConversionUtility.CleanYamlBeforeDeserializationV2(yaml); json = JSONSerialization.DeserializeStringToObject(processedYaml); } //Build up the GitHub object piece by piece GitHubActionsRoot gitHubActions = new GitHubActionsRoot(); GeneralProcessing gp = new GeneralProcessing(_verbose); if (json != null) { //Name if (json["name"] != null) { string nameYaml = json["name"].ToString(); gitHubActions.name = gp.ProcessNameV2(nameYaml); } //Trigger/PR/Schedules TriggerProcessing tp = new TriggerProcessing(_verbose); if (json["trigger"] != null) { string triggerYaml = json["trigger"].ToString(); triggerYaml = ConversionUtility.ProcessNoneJsonElement(triggerYaml); gitHubActions.on = tp.ProcessTriggerV2(triggerYaml); } if (json["pr"] != null) { string prYaml = json["pr"].ToString(); prYaml = ConversionUtility.ProcessNoneJsonElement(prYaml); GitHubActions.Trigger prTrigger = tp.ProcessPullRequestV2(prYaml); if (gitHubActions.on == null) { gitHubActions.on = prTrigger; } else { gitHubActions.on.pull_request = prTrigger.pull_request; } } if (json["schedules"] != null) { string schedulesYaml = json["schedules"].ToString(); GitHubActions.Trigger schedules = tp.ProcessSchedulesV2(schedulesYaml); if (gitHubActions.on == null) { gitHubActions.on = schedules; } else { gitHubActions.on.schedule = schedules.schedule; } } //Parameters & Variables string parametersYaml = json["parameters"]?.ToString(); string variablesYaml = json["variables"]?.ToString(); VariablesProcessing vp = new VariablesProcessing(_verbose); gitHubActions.env = vp.ProcessParametersAndVariablesV2(parametersYaml, variablesYaml); //Resources string resourcesYaml = json["resources"]?.ToString(); //Resource Pipelines if (resourcesYaml?.IndexOf("\"pipelines\"") >= 0) { gitHubActions.messages.Add("TODO: Resource pipelines conversion not yet done: https://github.com/samsmithnz/AzurePipelinesToGitHubActionsConverter/issues/8"); } //Resource Repositories if (resourcesYaml?.IndexOf("\"repositories\"") >= 0) { gitHubActions.messages.Add("TODO: Resource repositories conversion not yet done: https://github.com/samsmithnz/AzurePipelinesToGitHubActionsConverter/issues/8"); } //Resource Container if (resourcesYaml?.IndexOf("\"containers\"") >= 0) { gitHubActions.messages.Add("TODO: Container conversion not yet done, we need help!: https://github.com/samsmithnz/AzurePipelinesToGitHubActionsConverter/issues/39"); } //Strategy string strategyYaml = json["strategy"]?.ToString(); //If we have stages, convert them into jobs first: if (json["stages"] != null) { StagesProcessing sp = new StagesProcessing(_verbose); gitHubActions.jobs = sp.ProcessStagesV2(json["stages"], strategyYaml); } //If we don't have stages, but have jobs: else if (json["stages"] == null && json["jobs"] != null) { JobProcessing jp = new JobProcessing(_verbose); gitHubActions.jobs = jp.ProcessJobsV2(jp.ExtractAzurePipelinesJobsV2(json["jobs"], strategyYaml), gp.ExtractResourcesV2(resourcesYaml)); _matrixVariableName = jp.MatrixVariableName; } //Otherwise, if we don't have stages or jobs, we just have steps, and need to load them into a new job else if (json["stages"] == null && json["jobs"] == null) { //Pool string poolYaml = json["pool"]?.ToString(); //pool/demands if (poolYaml?.IndexOf("\"demands\":") >= 0) { gitHubActions.messages.Add("Note: GitHub Actions does not have a 'demands' command on 'runs-on' yet"); } //Steps string stepsYaml = json["steps"]?.ToString(); JobProcessing jp = new JobProcessing(_verbose); AzurePipelines.Job[] pipelineJobs = jp.ProcessJobFromPipelineRootV2(poolYaml, strategyYaml, stepsYaml); Resources resources = gp.ExtractResourcesV2(resourcesYaml); gitHubActions.jobs = jp.ProcessJobsV2(pipelineJobs, resources); _matrixVariableName = jp.MatrixVariableName; } if (gitHubActions.jobs != null && gitHubActions.jobs.Count == 0) { gitHubActions.messages.Add("Note that although having no jobs is valid YAML, it is not a valid GitHub Action."); } //Load in all variables. Duplicates are ok, they are processed the same variableList.AddRange(vp.SearchForVariables(yaml)); variableList.AddRange(vp.SearchForVariablesV2(gitHubActions)); //Create the GitHub YAML and apply some adjustments if (gitHubActions != null) { gitHubYaml = GitHubActionsSerialization.Serialize(gitHubActions, variableList, _matrixVariableName); } else { gitHubYaml = ""; } //Load failed task comments for processing //Add any header messages if (gitHubActions?.messages != null) { foreach (string message in gitHubActions.messages) { stepComments.Add(ConversionUtility.ConvertMessageToYamlComment(message)); } } if (gitHubActions?.jobs != null) { //Add each individual step comments foreach (KeyValuePair <string, GitHubActions.Job> job in gitHubActions.jobs) { if (job.Value.steps != null) { if (job.Value.job_message != null) { stepComments.Add(ConversionUtility.ConvertMessageToYamlComment(job.Value.job_message)); } foreach (GitHubActions.Step step in job.Value.steps) { if (step != null && string.IsNullOrEmpty(step.step_message) == false) { stepComments.Add(ConversionUtility.ConvertMessageToYamlComment(step.step_message)); } } } } } } //Append all of the comments to the top of the file foreach (string item in stepComments) { gitHubYaml = item + System.Environment.NewLine + gitHubYaml; } //Return the final conversion result, with the original (pipeline) yaml, processed (actions) yaml, and any comments return(new ConversionResponse { pipelinesYaml = yaml, actionsYaml = gitHubYaml, comments = stepComments, v2ConversionSuccessful = true }); }