/// <summary> /// Recursively find the controlling triggers for the given node. /// </summary> /// <param name="NodeToDo">Node to find the controlling triggers for</param> static void FindControllingTriggers(BuildNode Node) { if (Node.ControllingTriggers == null) { Node.ControllingTriggers = new TriggerNode[0]; // Find the immediate trigger controlling this one List <TriggerNode> PreviousTriggers = new List <TriggerNode>(); foreach (BuildNode Dependency in Node.OrderDependencies) { FindControllingTriggers(Dependency); TriggerNode PreviousTrigger = Dependency as TriggerNode; if (PreviousTrigger == null && Dependency.ControllingTriggers.Length > 0) { PreviousTrigger = Dependency.ControllingTriggers.Last(); } if (PreviousTrigger != null && !PreviousTriggers.Contains(PreviousTrigger)) { PreviousTriggers.Add(PreviousTrigger); } } // Remove previous triggers from the list that aren't the last in the chain. If there's a trigger chain of X.Y.Z, and a node has dependencies behind all three, the only trigger we care about is Z. PreviousTriggers.RemoveAll(x => PreviousTriggers.Any(y => y.ControllingTriggers.Contains(x))); // We only support one direct controlling trigger at the moment (though it may be in a chain with other triggers) if (PreviousTriggers.Count > 1) { throw new AutomationException("Node {0} has multiple controlling triggers: {1}", Node.Name, String.Join(", ", PreviousTriggers.Select(x => x.Name))); } // Update the list of controlling triggers if (PreviousTriggers.Count == 1) { List <TriggerNode> ControllingTriggers = new List <TriggerNode>(); ControllingTriggers.AddRange(PreviousTriggers[0].ControllingTriggers); ControllingTriggers.Add(PreviousTriggers[0]); Node.ControllingTriggers = ControllingTriggers.ToArray(); } } }
public void DoCommanderSetup(IEnumerable <BuildNode> AllNodes, IEnumerable <AggregateNode> AllAggregates, List <BuildNode> OrdereredToDo, List <BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, string CLString, TriggerNode ExplicitTrigger, List <TriggerNode> UnfinishedTriggers, string FakeFail) { List <AggregateNode> SeparatePromotables = FindPromotables(AllAggregates); Dictionary <BuildNode, List <AggregateNode> > DependentPromotions = FindDependentPromotables(AllNodes, SeparatePromotables); Dictionary <BuildNode, int> FullNodeListSortKey = GetDisplayOrder(SortedNodes); if (OrdereredToDo.Count == 0) { throw new AutomationException("No nodes to do!"); } List <string> ECProps = new List <string>(); ECProps.Add(String.Format("TimeIndex={0}", TimeIndex)); foreach (BuildNode Node in SortedNodes.Where(x => x.FrequencyShift != BuildNode.ExplicitFrequencyShift)) { ECProps.Add(string.Format("AllNodes/{0}={1}", Node.Name, GetNodeForAllNodesProperty(Node, TimeQuantum))); } foreach (KeyValuePair <BuildNode, int> NodePair in FullNodeListSortKey.Where(x => x.Key.FrequencyShift != BuildNode.ExplicitFrequencyShift)) { ECProps.Add(string.Format("SortKey/{0}={1}", NodePair.Key.Name, NodePair.Value)); } foreach (KeyValuePair <BuildNode, List <AggregateNode> > NodePair in DependentPromotions) { ECProps.Add(string.Format("DependentPromotions/{0}={1}", NodePair.Key.Name, String.Join(" ", NodePair.Value.Select(x => x.Name)))); } foreach (AggregateNode Node in SeparatePromotables) { ECProps.Add(string.Format("PossiblePromotables/{0}={1}", Node.Name, "")); } List <string> ECJobProps = new List <string>(); if (ExplicitTrigger != null) { ECJobProps.Add("IsRoot=0"); } else { ECJobProps.Add("IsRoot=1"); } // here we are just making sure everything before the explicit trigger is completed. if (ExplicitTrigger != null) { foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger)) // if something is already finished, we don't put it into EC { throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name); } } } BuildNode LastSticky = null; bool HitNonSticky = false; bool bHaveECNodes = false; List <string> StepList = new List <string>(); StepList.Add("use strict;"); StepList.Add("use diagnostics;"); StepList.Add("use ElectricCommander();"); StepList.Add("my $ec = new ElectricCommander;"); StepList.Add("$ec->setTimeout(600);"); StepList.Add("my $batch = $ec->newBatch(\"serial\");"); // sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { bHaveECNodes = true; if (NodeToDo.IsSticky) { LastSticky = NodeToDo; if (HitNonSticky && !bSkipTriggers) { throw new AutomationException("Sticky and non-sticky jobs did not sort right."); } } else { HitNonSticky = true; } } } using (CommandUtils.TelemetryStopwatch PerlOutputStopwatch = new CommandUtils.TelemetryStopwatch("PerlOutput")) { string ParentPath = Command.ParseParamValue("ParentPath"); string BaseArgs = String.Format("$batch->createJobStep({{parentPath => '{0}'", ParentPath); bool bHasNoop = false; if (LastSticky == null && bHaveECNodes) { // if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource string Args = String.Format("{0}, subprocedure => 'GUBP_UAT_Node', parallel => '0', jobStepName => 'Noop', actualParameter => [{{actualParameterName => 'NodeName', value => 'Noop'}}, {{actualParameterName => 'Sticky', value =>'1' }}], releaseMode => 'release'}});", BaseArgs); StepList.Add(Args); bHasNoop = true; } Dictionary <string, List <BuildNode> > AgentGroupChains = new Dictionary <string, List <BuildNode> >(); List <BuildNode> StickyChain = new List <BuildNode>(); foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { string MyAgentGroup = NodeToDo.AgentSharingGroup; if (MyAgentGroup != "") { if (!AgentGroupChains.ContainsKey(MyAgentGroup)) { AgentGroupChains.Add(MyAgentGroup, new List <BuildNode> { NodeToDo }); } else { AgentGroupChains[MyAgentGroup].Add(NodeToDo); } } } if (NodeToDo.IsSticky) { if (!StickyChain.Contains(NodeToDo)) { StickyChain.Add(NodeToDo); } } } foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { List <string> NodeProps = GetECPropsForNode(NodeToDo); ECProps.AddRange(NodeProps); bool Sticky = NodeToDo.IsSticky; if (NodeToDo.IsSticky) { if (NodeToDo.AgentSharingGroup != "") { throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name); } if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform); } if (NodeToDo.AgentRequirements != "") { throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name); } } string ProcedureInfix = ""; if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString(); } bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor; TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode; List <Tuple <string, string> > Parameters = new List <Tuple <string, string> >(); Parameters.Add(new Tuple <string, string>("NodeName", NodeToDo.Name)); Parameters.Add(new Tuple <string, string>("Sticky", NodeToDo.IsSticky ? "1" : "0")); if (NodeToDo.AgentSharingGroup != "") { Parameters.Add(new Tuple <string, string>("AgentSharingGroup", NodeToDo.AgentSharingGroup)); } string Procedure; if (TriggerNodeToDo == null || TriggerNodeToDo.IsTriggered) { if (NodeToDo.IsParallelAgentShareEditor) { Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor"; } else { Procedure = "GUBP" + ProcedureInfix + "_UAT_Node"; if (!NodeToDo.IsSticky) { Procedure += "_Parallel"; } if (NodeToDo.AgentSharingGroup != "") { Procedure += "_AgentShare"; } } if (NodeToDo.IsSticky && NodeToDo == LastSticky) { Procedure += "_Release"; } } else { if (TriggerNodeToDo.RequiresRecursiveWorkflow) { Procedure = "GUBP_UAT_Trigger"; //here we run a recursive workflow to wait for the trigger } else { Procedure = "GUBP_Hardcoded_Trigger"; //here we advance the state in the hardcoded workflow so folks can approve } Parameters.Add(new Tuple <string, string>("TriggerState", TriggerNodeToDo.StateName)); Parameters.Add(new Tuple <string, string>("ActionText", TriggerNodeToDo.ActionText)); Parameters.Add(new Tuple <string, string>("DescText", TriggerNodeToDo.DescriptionText)); if (NodeToDo.RecipientsForFailureEmails.Length > 0) { Parameters.Add(new Tuple <string, string>("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails))); } } string ActualParameterArgs = String.Join(", ", Parameters.Select(x => String.Format("{{actualParameterName => '{0}', value => '{1}'}}", x.Item1, x.Item2))); string Args = String.Format("{0}, subprocedure => '{1}', parallel => '{2}', jobStepName => '{3}', actualParameter => [{4}]", BaseArgs, Procedure, DoParallel? 1 : 0, NodeToDo.Name, ActualParameterArgs); List <BuildNode> UncompletedEcDeps = new List <BuildNode>(); { foreach (BuildNode Dep in NodeToDo.AllDirectDependencies) { if (!Dep.IsComplete && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC { if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(NodeToDo)) { throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name); } UncompletedEcDeps.Add(Dep); } } } string PreCondition = GetPreConditionForNode(OrdereredToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps); string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath); string MyAgentGroup = NodeToDo.AgentSharingGroup; bool bDoNestedJobstep = false; bool bDoFirstNestedJobstep = false; string NodeParentPath = ParentPath; if (MyAgentGroup != "") { bDoNestedJobstep = true; NodeParentPath = ParentPath + "/jobSteps[" + MyAgentGroup + "]"; List <BuildNode> MyChain = AgentGroupChains[MyAgentGroup]; int MyIndex = MyChain.IndexOf(NodeToDo); if (MyIndex <= 0) { bDoFirstNestedJobstep = bDoNestedJobstep; } } if (bDoNestedJobstep) { if (bDoFirstNestedJobstep) { { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}', jobStepName => '{1}', parallel => '1'", ParentPath, MyAgentGroup); if (!String.IsNullOrEmpty(PreCondition)) { NestArgs = NestArgs + ", precondition => " + PreCondition; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetPool', subprocedure => 'GUBP{3}_AgentShare_GetPool', parallel => '1', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{4}'}}, {{actualParameterName => 'NodeName', value => '{5}'}}]", ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, MyAgentGroup, NodeToDo.Name); if (!String.IsNullOrEmpty(PreCondition)) { NestArgs = NestArgs + ", precondition => " + PreCondition; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetAgent', subprocedure => 'GUBP{3}_AgentShare_GetAgent', parallel => '1', exclusiveMode => 'call', resourceName => '{4}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{5}'}}, {{actualParameterName => 'NodeName', value=> '{6}'}}]", ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", MyAgentGroup), MyAgentGroup, NodeToDo.Name); { NestArgs = NestArgs + ", precondition => "; NestArgs = NestArgs + "\"\\$\" . \"[/javascript if("; NestArgs = NestArgs + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetPool]/status') == 'completed'"; NestArgs = NestArgs + ") true;]\""; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { PreCondition = "\"\\$\" . \"[/javascript if("; PreCondition = PreCondition + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetAgent]/status') == 'completed'"; PreCondition = PreCondition + ") true;]\""; } } Args = Args.Replace(String.Format("parentPath => '{0}'", ParentPath), String.Format("parentPath => '{0}'", NodeParentPath)); Args = Args.Replace("UAT_Node_Parallel_AgentShare", "UAT_Node_Parallel_AgentShare3"); } if (!String.IsNullOrEmpty(PreCondition)) { Args = Args + ", precondition => " + PreCondition; } if (!String.IsNullOrEmpty(RunCondition)) { Args = Args + ", condition => " + RunCondition; } #if false // this doesn't work because it includes precondition time if (GUBPNodes[NodeToDo].TimeoutInMinutes() > 0) { Args = Args + String.Format(" --timeLimitUnits minutes --timeLimit {0}", GUBPNodes[NodeToDo].TimeoutInMinutes()); } #endif if (Sticky && NodeToDo == LastSticky) { Args = Args + ", releaseMode => 'release'"; } Args = Args + "});"; StepList.Add(Args); if (MyAgentGroup != "" && !bDoNestedJobstep) { List <BuildNode> MyChain = AgentGroupChains[MyAgentGroup]; int MyIndex = MyChain.IndexOf(NodeToDo); if (MyIndex == MyChain.Count - 1) { string RelPreCondition = "\"\\$\" . \"[/javascript if("; // this runs "parallel", but we a precondition to serialize it RelPreCondition = RelPreCondition + "getProperty('" + ParentPath + "/jobSteps[" + NodeToDo.Name + "]/status') == 'completed'"; RelPreCondition = RelPreCondition + ") true;]\""; // we need to release the resource string RelArgs = String.Format("{0}, subprocedure => 'GUBP_Release_AgentShare', parallel => '1', jobStepName => 'Release_{1}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', valued => '{2}'}}], releaseMode => 'release', precondition => '{3}'", BaseArgs, MyAgentGroup, MyAgentGroup, RelPreCondition); StepList.Add(RelArgs); } } } } WriteECPerl(StepList); } bool bHasTests = OrdereredToDo.Any(x => x.Node.IsTest()); RunECTool(String.Format("setProperty \"/myWorkflow/HasTests\" \"{0}\"", bHasTests)); { ECProps.Add("GUBP_LoadedProps=1"); string BranchDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchDef.properties"); CommandUtils.WriteAllLines(BranchDefFile, ECProps.ToArray()); RunECTool(String.Format("setProperty \"/myWorkflow/BranchDefFile\" \"{0}\"", BranchDefFile.Replace("\\", "\\\\"))); } { ECProps.Add("GUBP_LoadedJobProps=1"); string BranchJobDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchJobDef.properties"); CommandUtils.WriteAllLines(BranchJobDefFile, ECProps.ToArray()); RunECTool(String.Format("setProperty \"/myJob/BranchJobDefFile\" \"{0}\"", BranchJobDefFile.Replace("\\", "\\\\"))); } }
private void WriteJobSteps(List <BuildNode> OrderedToDo, bool bSkipTriggers) { BuildNode LastSticky = null; bool HitNonSticky = false; bool bHaveECNodes = false; // sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs foreach (BuildNode NodeToDo in OrderedToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { bHaveECNodes = true; if (NodeToDo.IsSticky) { LastSticky = NodeToDo; if (HitNonSticky && !bSkipTriggers) { throw new AutomationException("Sticky and non-sticky jobs did not sort right."); } } else { HitNonSticky = true; } } } List <JobStep> Steps = new List <JobStep>(); using (TelemetryStopwatch PerlOutputStopwatch = new TelemetryStopwatch("PerlOutput")) { string ParentPath = Command.ParseParamValue("ParentPath"); bool bHasNoop = false; if (LastSticky == null && bHaveECNodes) { // if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource JobStep NoopStep = new JobStep(ParentPath, "Noop", "GUBP_UAT_Node", false, null, null, JobStepReleaseMode.Release); NoopStep.ActualParameters.Add("NodeName", "Noop"); NoopStep.ActualParameters.Add("Sticky", "1"); Steps.Add(NoopStep); bHasNoop = true; } Dictionary <string, List <BuildNode> > AgentGroupChains = new Dictionary <string, List <BuildNode> >(); List <BuildNode> StickyChain = new List <BuildNode>(); foreach (BuildNode NodeToDo in OrderedToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { string MyAgentGroup = NodeToDo.AgentSharingGroup; if (MyAgentGroup != "") { if (!AgentGroupChains.ContainsKey(MyAgentGroup)) { AgentGroupChains.Add(MyAgentGroup, new List <BuildNode> { NodeToDo }); } else { AgentGroupChains[MyAgentGroup].Add(NodeToDo); } } } if (NodeToDo.IsSticky) { if (!StickyChain.Contains(NodeToDo)) { StickyChain.Add(NodeToDo); } } } foreach (BuildNode NodeToDo in OrderedToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { bool Sticky = NodeToDo.IsSticky; if (NodeToDo.IsSticky) { if (NodeToDo.AgentSharingGroup != "") { throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name); } if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform); } if (NodeToDo.AgentRequirements != "") { throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name); } } string ProcedureInfix = ""; if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString(); } bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor; List <BuildNode> UncompletedEcDeps = new List <BuildNode>(); foreach (BuildNode Dep in FindDirectOrderDependencies(NodeToDo)) { if (!Dep.IsComplete && OrderedToDo.Contains(Dep)) // if something is already finished, we don't put it into EC { if (OrderedToDo.IndexOf(Dep) > OrderedToDo.IndexOf(NodeToDo)) { throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name); } UncompletedEcDeps.Add(Dep); } } string PreCondition = GetPreConditionForNode(OrderedToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps); string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath); // Create the job steps for this node TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode; if (TriggerNodeToDo == null) { // Create the jobs to setup the agent sharing group if necessary string NodeParentPath = ParentPath; if (NodeToDo.AgentSharingGroup != "") { NodeParentPath = String.Format("{0}/jobSteps[{1}]", NodeParentPath, NodeToDo.AgentSharingGroup); List <BuildNode> MyChain = AgentGroupChains[NodeToDo.AgentSharingGroup]; if (MyChain.IndexOf(NodeToDo) <= 0) { // Create the parent job step for this group JobStep ParentStep = new JobStep(ParentPath, NodeToDo.AgentSharingGroup, null, true, PreCondition, null, JobStepReleaseMode.Keep); Steps.Add(ParentStep); // Get the resource pool JobStep GetPoolStep = new JobStep(NodeParentPath, String.Format("{0}_GetPool", NodeToDo.AgentSharingGroup), String.Format("GUBP{0}_AgentShare_GetPool", ProcedureInfix), true, PreCondition, null, JobStepReleaseMode.Keep); GetPoolStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup); GetPoolStep.ActualParameters.Add("NodeName", NodeToDo.Name); Steps.Add(GetPoolStep); // Get the agent for this sharing group JobStep GetAgentStep = new JobStep(NodeParentPath, String.Format("{0}_GetAgent", NodeToDo.AgentSharingGroup), String.Format("GUBP{0}_AgentShare_GetAgent", ProcedureInfix), true, GetPoolStep.GetCompletedCondition(), null, JobStepReleaseMode.Keep); GetAgentStep.Exclusive = JobStepExclusiveMode.Call; GetAgentStep.ResourceName = String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", NodeToDo.AgentSharingGroup); GetAgentStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup); GetAgentStep.ActualParameters.Add("NodeName", NodeToDo.Name); Steps.Add(GetAgentStep); // Set the precondition from this point onwards to be whether the group was set up, since it can't succeed unless the original precondition succeeded PreCondition = GetAgentStep.GetCompletedCondition(); } } // Get the procedure name string Procedure; if (NodeToDo.IsParallelAgentShareEditor) { if (NodeToDo.AgentSharingGroup == "") { Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor"; } else { Procedure = "GUBP_UAT_Node_Parallel_AgentShare3_Editor"; } } else { if (NodeToDo.IsSticky) { Procedure = "GUBP" + ProcedureInfix + "_UAT_Node"; } else if (NodeToDo.AgentSharingGroup == "") { Procedure = String.Format("GUBP{0}_UAT_Node_Parallel", ProcedureInfix); } else { Procedure = String.Format("GUBP{0}_UAT_Node_Parallel_AgentShare3", ProcedureInfix); } } if (NodeToDo.IsSticky && NodeToDo == LastSticky) { Procedure += "_Release"; } // Build the job step for this node JobStep MainStep = new JobStep(NodeParentPath, NodeToDo.Name, Procedure, DoParallel, PreCondition, RunCondition, (NodeToDo.IsSticky && NodeToDo == LastSticky) ? JobStepReleaseMode.Release : JobStepReleaseMode.Keep); MainStep.ActualParameters.Add("NodeName", NodeToDo.Name); MainStep.ActualParameters.Add("Sticky", NodeToDo.IsSticky ? "1" : "0"); if (NodeToDo.AgentSharingGroup != "") { MainStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup); } Steps.Add(MainStep); } else { // Get the procedure name string Procedure; if (TriggerNodeToDo.IsTriggered) { Procedure = String.Format("GUBP{0}_UAT_Node{1}", ProcedureInfix, (NodeToDo == LastSticky) ? "_Release" : ""); } else if (TriggerNodeToDo.RequiresRecursiveWorkflow) { Procedure = "GUBP_UAT_Trigger"; //here we run a recursive workflow to wait for the trigger } else { Procedure = "GUBP_Hardcoded_Trigger"; //here we advance the state in the hardcoded workflow so folks can approve } // Create the job step JobStep TriggerStep = new JobStep(ParentPath, NodeToDo.Name, Procedure, DoParallel, PreCondition, RunCondition, (NodeToDo.IsSticky && NodeToDo == LastSticky) ? JobStepReleaseMode.Release : JobStepReleaseMode.Keep); TriggerStep.ActualParameters.Add("NodeName", NodeToDo.Name); TriggerStep.ActualParameters.Add("Sticky", NodeToDo.IsSticky ? "1" : "0"); if (!TriggerNodeToDo.IsTriggered) { TriggerStep.ActualParameters.Add("TriggerState", TriggerNodeToDo.StateName); TriggerStep.ActualParameters.Add("ActionText", TriggerNodeToDo.ActionText); TriggerStep.ActualParameters.Add("DescText", TriggerNodeToDo.DescriptionText); if (NodeToDo.RecipientsForFailureEmails.Length > 0) { TriggerStep.ActualParameters.Add("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails)); } } Steps.Add(TriggerStep); } } } WriteECPerl(Steps); } }
private void WriteJson(List <BuildNode> OrderedToDo, bool bSkipTriggers) { using (JsonWriter JsonWriter = new JsonWriter(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "JobSteps.json"))) { JsonWriter.WriteObjectStart(); JsonWriter.WriteArrayStart("JobSteps"); string CurrentAgentGroup = ""; foreach (BuildNode NodeToDo in OrderedToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { // Write the agent group object string AgentGroup = GetAgentGroupOrSticky(NodeToDo); if (AgentGroup != CurrentAgentGroup) { if (CurrentAgentGroup != "") { JsonWriter.WriteArrayEnd(); JsonWriter.WriteObjectEnd(); } CurrentAgentGroup = AgentGroup; if (CurrentAgentGroup != "") { JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Group", AgentGroup); JsonWriter.WriteValue("AgentPlatform", NodeToDo.AgentPlatform.ToString()); JsonWriter.WriteArrayStart("JobSteps"); } } // Add this node JsonWriter.WriteObjectStart(); JsonWriter.WriteValue("Node", NodeToDo.Name); if (CurrentAgentGroup == "") { JsonWriter.WriteValue("AgentPlatform", NodeToDo.AgentPlatform.ToString()); } if (!String.IsNullOrEmpty(NodeToDo.AgentRequirements)) { JsonWriter.WriteValue("AgentRequirements", NodeToDo.AgentRequirements); } // Write all the dependencies StringBuilder DependsOnList = new StringBuilder(); foreach (BuildNode Dep in FindDirectOrderDependencies(NodeToDo)) { if (!Dep.IsComplete && OrderedToDo.Contains(Dep)) // if something is already finished, we don't put it into EC { if (OrderedToDo.IndexOf(Dep) > OrderedToDo.IndexOf(NodeToDo)) { throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name); } if (DependsOnList.Length > 0) { DependsOnList.Append(";"); } string DepAgentGroup = GetAgentGroupOrSticky(Dep); if (DepAgentGroup != "") { DependsOnList.AppendFormat("{0}/", DepAgentGroup); } DependsOnList.Append(Dep.Name); } } JsonWriter.WriteValue("DependsOn", DependsOnList.ToString()); // Add any trigger-specific settings TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode; if (TriggerNodeToDo != null && !TriggerNodeToDo.IsTriggered) { JsonWriter.WriteValue("TriggerState", TriggerNodeToDo.StateName); JsonWriter.WriteValue("TriggerActionText", TriggerNodeToDo.ActionText); JsonWriter.WriteValue("TriggerDescriptionText", TriggerNodeToDo.DescriptionText); if (NodeToDo.RecipientsForFailureEmails.Length > 0) { JsonWriter.WriteValue("TriggerEmailRecipients", String.Join(" ", NodeToDo.RecipientsForFailureEmails)); } } JsonWriter.WriteObjectEnd(); } } if (CurrentAgentGroup != "") { JsonWriter.WriteArrayEnd(); JsonWriter.WriteObjectEnd(); } JsonWriter.WriteArrayEnd(); JsonWriter.WriteObjectEnd(); } }
public void DoCommanderSetup(IEnumerable <BuildNode> AllNodes, IEnumerable <AggregateNode> AllAggregates, List <BuildNode> OrderedToDo, List <BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, TriggerNode ExplicitTrigger, List <TriggerNode> UnfinishedTriggers) { // Check there's some nodes in the graph if (OrderedToDo.Count == 0) { throw new AutomationException("No nodes to do!"); } // Make sure everything before the explicit trigger is completed. if (ExplicitTrigger != null) { foreach (BuildNode NodeToDo in OrderedToDo) { if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger)) // if something is already finished, we don't put it into EC { throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name); } } } // Update all the EC properties, and the branch definition files WriteProperties(AllNodes, AllAggregates, OrderedToDo, SortedNodes, TimeIndex, TimeQuantum); // Write all the job setup WriteJobSteps(OrderedToDo, bSkipTriggers); // Export it as JSON too WriteJson(OrderedToDo, bSkipTriggers); }
public void DoCommanderSetup(IEnumerable<BuildNode> AllNodes, IEnumerable<AggregateNode> AllAggregates, List<BuildNode> OrderedToDo, List<BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, TriggerNode ExplicitTrigger, List<TriggerNode> UnfinishedTriggers, string FakeFail) { // Check there's some nodes in the graph if (OrderedToDo.Count == 0) { throw new AutomationException("No nodes to do!"); } // Make sure everything before the explicit trigger is completed. if (ExplicitTrigger != null) { foreach (BuildNode NodeToDo in OrderedToDo) { if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger)) // if something is already finished, we don't put it into EC { throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name); } } } // Update all the EC properties, and the branch definition files WriteProperties(AllNodes, AllAggregates, OrderedToDo, SortedNodes, TimeIndex, TimeQuantum); // Write all the job setup WriteJobSteps(OrderedToDo, bSkipTriggers); }
public void DoCommanderSetup(IEnumerable<BuildNode> AllNodes, IEnumerable<AggregateNode> AllAggregates, List<BuildNode> OrdereredToDo, List<BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, string CLString, TriggerNode ExplicitTrigger, List<TriggerNode> UnfinishedTriggers, string FakeFail) { List<AggregateNode> SeparatePromotables = FindPromotables(AllAggregates); Dictionary<BuildNode, List<AggregateNode>> DependentPromotions = FindDependentPromotables(AllNodes, SeparatePromotables); Dictionary<BuildNode, int> FullNodeListSortKey = GetDisplayOrder(SortedNodes); if (OrdereredToDo.Count == 0) { throw new AutomationException("No nodes to do!"); } List<string> ECProps = new List<string>(); ECProps.Add(String.Format("TimeIndex={0}", TimeIndex)); foreach (BuildNode Node in SortedNodes.Where(x => x.FrequencyShift != BuildNode.ExplicitFrequencyShift)) { ECProps.Add(string.Format("AllNodes/{0}={1}", Node.Name, GetNodeForAllNodesProperty(Node, TimeQuantum))); } foreach (KeyValuePair<BuildNode, int> NodePair in FullNodeListSortKey.Where(x => x.Key.FrequencyShift != BuildNode.ExplicitFrequencyShift)) { ECProps.Add(string.Format("SortKey/{0}={1}", NodePair.Key.Name, NodePair.Value)); } foreach (KeyValuePair<BuildNode, List<AggregateNode>> NodePair in DependentPromotions) { ECProps.Add(string.Format("DependentPromotions/{0}={1}", NodePair.Key.Name, String.Join(" ", NodePair.Value.Select(x => x.Name)))); } foreach (AggregateNode Node in SeparatePromotables) { ECProps.Add(string.Format("PossiblePromotables/{0}={1}", Node.Name, "")); } List<string> ECJobProps = new List<string>(); if (ExplicitTrigger != null) { ECJobProps.Add("IsRoot=0"); } else { ECJobProps.Add("IsRoot=1"); } // here we are just making sure everything before the explicit trigger is completed. if (ExplicitTrigger != null) { foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger)) // if something is already finished, we don't put it into EC { throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name); } } } BuildNode LastSticky = null; bool HitNonSticky = false; bool bHaveECNodes = false; List<string> StepList = new List<string>(); StepList.Add("use strict;"); StepList.Add("use diagnostics;"); StepList.Add("use ElectricCommander();"); StepList.Add("my $ec = new ElectricCommander;"); StepList.Add("$ec->setTimeout(600);"); StepList.Add("my $batch = $ec->newBatch(\"serial\");"); // sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { bHaveECNodes = true; if (NodeToDo.IsSticky) { LastSticky = NodeToDo; if (HitNonSticky && !bSkipTriggers) { throw new AutomationException("Sticky and non-sticky jobs did not sort right."); } } else { HitNonSticky = true; } } } using(CommandUtils.TelemetryStopwatch PerlOutputStopwatch = new CommandUtils.TelemetryStopwatch("PerlOutput")) { string ParentPath = Command.ParseParamValue("ParentPath"); string BaseArgs = String.Format("$batch->createJobStep({{parentPath => '{0}'", ParentPath); bool bHasNoop = false; if (LastSticky == null && bHaveECNodes) { // if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource string Args = String.Format("{0}, subprocedure => 'GUBP_UAT_Node', parallel => '0', jobStepName => 'Noop', actualParameter => [{{actualParameterName => 'NodeName', value => 'Noop'}}, {{actualParameterName => 'Sticky', value =>'1' }}], releaseMode => 'release'}});", BaseArgs); StepList.Add(Args); bHasNoop = true; } Dictionary<string, List<BuildNode>> AgentGroupChains = new Dictionary<string, List<BuildNode>>(); List<BuildNode> StickyChain = new List<BuildNode>(); foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { string MyAgentGroup = NodeToDo.AgentSharingGroup; if (MyAgentGroup != "") { if (!AgentGroupChains.ContainsKey(MyAgentGroup)) { AgentGroupChains.Add(MyAgentGroup, new List<BuildNode> { NodeToDo }); } else { AgentGroupChains[MyAgentGroup].Add(NodeToDo); } } } if (NodeToDo.IsSticky) { if (!StickyChain.Contains(NodeToDo)) { StickyChain.Add(NodeToDo); } } } foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { List<string> NodeProps = GetECPropsForNode(NodeToDo); ECProps.AddRange(NodeProps); bool Sticky = NodeToDo.IsSticky; if(NodeToDo.IsSticky) { if(NodeToDo.AgentSharingGroup != "") { throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name); } if(NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform); } if(NodeToDo.AgentRequirements != "") { throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name); } } string ProcedureInfix = ""; if(NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString(); } bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor; TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode; List<Tuple<string, string>> Parameters = new List<Tuple<string,string>>(); Parameters.Add(new Tuple<string, string>("NodeName", NodeToDo.Name)); Parameters.Add(new Tuple<string, string>("Sticky", NodeToDo.IsSticky ? "1" : "0")); if (NodeToDo.AgentSharingGroup != "") { Parameters.Add(new Tuple<string, string>("AgentSharingGroup", NodeToDo.AgentSharingGroup)); } string Procedure; if(TriggerNodeToDo == null || TriggerNodeToDo.IsTriggered) { if (NodeToDo.IsParallelAgentShareEditor) { Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor"; } else { Procedure = "GUBP" + ProcedureInfix + "_UAT_Node"; if (!NodeToDo.IsSticky) { Procedure += "_Parallel"; } if (NodeToDo.AgentSharingGroup != "") { Procedure += "_AgentShare"; } } if (NodeToDo.IsSticky && NodeToDo == LastSticky) { Procedure += "_Release"; } } else { if(TriggerNodeToDo.RequiresRecursiveWorkflow) { Procedure = "GUBP_UAT_Trigger"; //here we run a recursive workflow to wait for the trigger } else { Procedure = "GUBP_Hardcoded_Trigger"; //here we advance the state in the hardcoded workflow so folks can approve } Parameters.Add(new Tuple<string, string>("TriggerState", TriggerNodeToDo.StateName)); Parameters.Add(new Tuple<string, string>("ActionText", TriggerNodeToDo.ActionText)); Parameters.Add(new Tuple<string, string>("DescText", TriggerNodeToDo.DescriptionText)); if (NodeToDo.RecipientsForFailureEmails.Length > 0) { Parameters.Add(new Tuple<string, string>("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails))); } } string ActualParameterArgs = String.Join(", ", Parameters.Select(x => String.Format("{{actualParameterName => '{0}', value => '{1}'}}", x.Item1, x.Item2))); string Args = String.Format("{0}, subprocedure => '{1}', parallel => '{2}', jobStepName => '{3}', actualParameter => [{4}]", BaseArgs, Procedure, DoParallel? 1 : 0, NodeToDo.Name, ActualParameterArgs); List<BuildNode> UncompletedEcDeps = new List<BuildNode>(); { foreach (BuildNode Dep in NodeToDo.AllDirectDependencies) { if (!Dep.IsComplete && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC { if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(NodeToDo)) { throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name); } UncompletedEcDeps.Add(Dep); } } } string PreCondition = GetPreConditionForNode(OrdereredToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps); string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath); string MyAgentGroup = NodeToDo.AgentSharingGroup; bool bDoNestedJobstep = false; bool bDoFirstNestedJobstep = false; string NodeParentPath = ParentPath; if (MyAgentGroup != "") { bDoNestedJobstep = true; NodeParentPath = ParentPath + "/jobSteps[" + MyAgentGroup + "]"; List<BuildNode> MyChain = AgentGroupChains[MyAgentGroup]; int MyIndex = MyChain.IndexOf(NodeToDo); if (MyIndex <= 0) { bDoFirstNestedJobstep = bDoNestedJobstep; } } if (bDoNestedJobstep) { if (bDoFirstNestedJobstep) { { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}', jobStepName => '{1}', parallel => '1'", ParentPath, MyAgentGroup); if (!String.IsNullOrEmpty(PreCondition)) { NestArgs = NestArgs + ", precondition => " + PreCondition; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetPool', subprocedure => 'GUBP{3}_AgentShare_GetPool', parallel => '1', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{4}'}}, {{actualParameterName => 'NodeName', value => '{5}'}}]", ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, MyAgentGroup, NodeToDo.Name); if (!String.IsNullOrEmpty(PreCondition)) { NestArgs = NestArgs + ", precondition => " + PreCondition; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetAgent', subprocedure => 'GUBP{3}_AgentShare_GetAgent', parallel => '1', exclusiveMode => 'call', resourceName => '{4}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{5}'}}, {{actualParameterName => 'NodeName', value=> '{6}'}}]", ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", MyAgentGroup), MyAgentGroup, NodeToDo.Name); { NestArgs = NestArgs + ", precondition => "; NestArgs = NestArgs + "\"\\$\" . \"[/javascript if("; NestArgs = NestArgs + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetPool]/status') == 'completed'"; NestArgs = NestArgs + ") true;]\""; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { PreCondition = "\"\\$\" . \"[/javascript if("; PreCondition = PreCondition + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetAgent]/status') == 'completed'"; PreCondition = PreCondition + ") true;]\""; } } Args = Args.Replace(String.Format("parentPath => '{0}'", ParentPath), String.Format("parentPath => '{0}'", NodeParentPath)); Args = Args.Replace("UAT_Node_Parallel_AgentShare", "UAT_Node_Parallel_AgentShare3"); } if (!String.IsNullOrEmpty(PreCondition)) { Args = Args + ", precondition => " + PreCondition; } if (!String.IsNullOrEmpty(RunCondition)) { Args = Args + ", condition => " + RunCondition; } #if false // this doesn't work because it includes precondition time if (GUBPNodes[NodeToDo].TimeoutInMinutes() > 0) { Args = Args + String.Format(" --timeLimitUnits minutes --timeLimit {0}", GUBPNodes[NodeToDo].TimeoutInMinutes()); } #endif if (Sticky && NodeToDo == LastSticky) { Args = Args + ", releaseMode => 'release'"; } Args = Args + "});"; StepList.Add(Args); if (MyAgentGroup != "" && !bDoNestedJobstep) { List<BuildNode> MyChain = AgentGroupChains[MyAgentGroup]; int MyIndex = MyChain.IndexOf(NodeToDo); if (MyIndex == MyChain.Count - 1) { string RelPreCondition = "\"\\$\" . \"[/javascript if("; // this runs "parallel", but we a precondition to serialize it RelPreCondition = RelPreCondition + "getProperty('" + ParentPath + "/jobSteps[" + NodeToDo.Name + "]/status') == 'completed'"; RelPreCondition = RelPreCondition + ") true;]\""; // we need to release the resource string RelArgs = String.Format("{0}, subprocedure => 'GUBP_Release_AgentShare', parallel => '1', jobStepName => 'Release_{1}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', valued => '{2}'}}], releaseMode => 'release', precondition => '{3}'", BaseArgs, MyAgentGroup, MyAgentGroup, RelPreCondition); StepList.Add(RelArgs); } } } } WriteECPerl(StepList); } bool bHasTests = OrdereredToDo.Any(x => x.Node.IsTest()); RunECTool(String.Format("setProperty \"/myWorkflow/HasTests\" \"{0}\"", bHasTests)); { ECProps.Add("GUBP_LoadedProps=1"); string BranchDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchDef.properties"); CommandUtils.WriteAllLines(BranchDefFile, ECProps.ToArray()); RunECTool(String.Format("setProperty \"/myWorkflow/BranchDefFile\" \"{0}\"", BranchDefFile.Replace("\\", "\\\\"))); } { ECProps.Add("GUBP_LoadedJobProps=1"); string BranchJobDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchJobDef.properties"); CommandUtils.WriteAllLines(BranchJobDefFile, ECProps.ToArray()); RunECTool(String.Format("setProperty \"/myJob/BranchJobDefFile\" \"{0}\"", BranchJobDefFile.Replace("\\", "\\\\"))); } }