/// <summary>
        /// Recursively find the controlling triggers for the given node.
        /// </summary>
        /// <param name="NodeToDo">Node to find the controlling triggers for</param>
        static void FindControllingTriggers(BuildNode Node)
        {
            if (Node.ControllingTriggers == null)
            {
                Node.ControllingTriggers = new TriggerNode[0];

                // Find the immediate trigger controlling this one
                List <TriggerNode> PreviousTriggers = new List <TriggerNode>();
                foreach (BuildNode Dependency in Node.OrderDependencies)
                {
                    FindControllingTriggers(Dependency);

                    TriggerNode PreviousTrigger = Dependency as TriggerNode;
                    if (PreviousTrigger == null && Dependency.ControllingTriggers.Length > 0)
                    {
                        PreviousTrigger = Dependency.ControllingTriggers.Last();
                    }

                    if (PreviousTrigger != null && !PreviousTriggers.Contains(PreviousTrigger))
                    {
                        PreviousTriggers.Add(PreviousTrigger);
                    }
                }

                // Remove previous triggers from the list that aren't the last in the chain. If there's a trigger chain of X.Y.Z, and a node has dependencies behind all three, the only trigger we care about is Z.
                PreviousTriggers.RemoveAll(x => PreviousTriggers.Any(y => y.ControllingTriggers.Contains(x)));

                // We only support one direct controlling trigger at the moment (though it may be in a chain with other triggers)
                if (PreviousTriggers.Count > 1)
                {
                    throw new AutomationException("Node {0} has multiple controlling triggers: {1}", Node.Name, String.Join(", ", PreviousTriggers.Select(x => x.Name)));
                }

                // Update the list of controlling triggers
                if (PreviousTriggers.Count == 1)
                {
                    List <TriggerNode> ControllingTriggers = new List <TriggerNode>();
                    ControllingTriggers.AddRange(PreviousTriggers[0].ControllingTriggers);
                    ControllingTriggers.Add(PreviousTriggers[0]);
                    Node.ControllingTriggers = ControllingTriggers.ToArray();
                }
            }
        }
        public void DoCommanderSetup(IEnumerable <BuildNode> AllNodes, IEnumerable <AggregateNode> AllAggregates, List <BuildNode> OrdereredToDo, List <BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, string CLString, TriggerNode ExplicitTrigger, List <TriggerNode> UnfinishedTriggers, string FakeFail)
        {
            List <AggregateNode> SeparatePromotables = FindPromotables(AllAggregates);
            Dictionary <BuildNode, List <AggregateNode> > DependentPromotions = FindDependentPromotables(AllNodes, SeparatePromotables);

            Dictionary <BuildNode, int> FullNodeListSortKey = GetDisplayOrder(SortedNodes);

            if (OrdereredToDo.Count == 0)
            {
                throw new AutomationException("No nodes to do!");
            }
            List <string> ECProps = new List <string>();

            ECProps.Add(String.Format("TimeIndex={0}", TimeIndex));
            foreach (BuildNode Node in SortedNodes.Where(x => x.FrequencyShift != BuildNode.ExplicitFrequencyShift))
            {
                ECProps.Add(string.Format("AllNodes/{0}={1}", Node.Name, GetNodeForAllNodesProperty(Node, TimeQuantum)));
            }
            foreach (KeyValuePair <BuildNode, int> NodePair in FullNodeListSortKey.Where(x => x.Key.FrequencyShift != BuildNode.ExplicitFrequencyShift))
            {
                ECProps.Add(string.Format("SortKey/{0}={1}", NodePair.Key.Name, NodePair.Value));
            }
            foreach (KeyValuePair <BuildNode, List <AggregateNode> > NodePair in DependentPromotions)
            {
                ECProps.Add(string.Format("DependentPromotions/{0}={1}", NodePair.Key.Name, String.Join(" ", NodePair.Value.Select(x => x.Name))));
            }
            foreach (AggregateNode Node in SeparatePromotables)
            {
                ECProps.Add(string.Format("PossiblePromotables/{0}={1}", Node.Name, ""));
            }
            List <string> ECJobProps = new List <string>();

            if (ExplicitTrigger != null)
            {
                ECJobProps.Add("IsRoot=0");
            }
            else
            {
                ECJobProps.Add("IsRoot=1");
            }

            // here we are just making sure everything before the explicit trigger is completed.
            if (ExplicitTrigger != null)
            {
                foreach (BuildNode NodeToDo in OrdereredToDo)
                {
                    if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger))                     // if something is already finished, we don't put it into EC
                    {
                        throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name);
                    }
                }
            }

            BuildNode     LastSticky   = null;
            bool          HitNonSticky = false;
            bool          bHaveECNodes = false;
            List <string> StepList     = new List <string>();

            StepList.Add("use strict;");
            StepList.Add("use diagnostics;");
            StepList.Add("use ElectricCommander();");
            StepList.Add("my $ec = new ElectricCommander;");
            StepList.Add("$ec->setTimeout(600);");
            StepList.Add("my $batch = $ec->newBatch(\"serial\");");
            // sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs
            foreach (BuildNode NodeToDo in OrdereredToDo)
            {
                if (!NodeToDo.IsComplete)                 // if something is already finished, we don't put it into EC
                {
                    bHaveECNodes = true;
                    if (NodeToDo.IsSticky)
                    {
                        LastSticky = NodeToDo;
                        if (HitNonSticky && !bSkipTriggers)
                        {
                            throw new AutomationException("Sticky and non-sticky jobs did not sort right.");
                        }
                    }
                    else
                    {
                        HitNonSticky = true;
                    }
                }
            }

            using (CommandUtils.TelemetryStopwatch PerlOutputStopwatch = new CommandUtils.TelemetryStopwatch("PerlOutput"))
            {
                string ParentPath = Command.ParseParamValue("ParentPath");
                string BaseArgs   = String.Format("$batch->createJobStep({{parentPath => '{0}'", ParentPath);

                bool bHasNoop = false;
                if (LastSticky == null && bHaveECNodes)
                {
                    // if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource
                    string Args = String.Format("{0}, subprocedure => 'GUBP_UAT_Node', parallel => '0', jobStepName => 'Noop', actualParameter => [{{actualParameterName => 'NodeName', value => 'Noop'}}, {{actualParameterName => 'Sticky', value =>'1' }}], releaseMode => 'release'}});", BaseArgs);
                    StepList.Add(Args);
                    bHasNoop = true;
                }

                Dictionary <string, List <BuildNode> > AgentGroupChains = new Dictionary <string, List <BuildNode> >();
                List <BuildNode> StickyChain = new List <BuildNode>();
                foreach (BuildNode NodeToDo in OrdereredToDo)
                {
                    if (!NodeToDo.IsComplete)                     // if something is already finished, we don't put it into EC
                    {
                        string MyAgentGroup = NodeToDo.AgentSharingGroup;
                        if (MyAgentGroup != "")
                        {
                            if (!AgentGroupChains.ContainsKey(MyAgentGroup))
                            {
                                AgentGroupChains.Add(MyAgentGroup, new List <BuildNode> {
                                    NodeToDo
                                });
                            }
                            else
                            {
                                AgentGroupChains[MyAgentGroup].Add(NodeToDo);
                            }
                        }
                    }
                    if (NodeToDo.IsSticky)
                    {
                        if (!StickyChain.Contains(NodeToDo))
                        {
                            StickyChain.Add(NodeToDo);
                        }
                    }
                }
                foreach (BuildNode NodeToDo in OrdereredToDo)
                {
                    if (!NodeToDo.IsComplete)                     // if something is already finished, we don't put it into EC
                    {
                        List <string> NodeProps = GetECPropsForNode(NodeToDo);
                        ECProps.AddRange(NodeProps);

                        bool Sticky = NodeToDo.IsSticky;
                        if (NodeToDo.IsSticky)
                        {
                            if (NodeToDo.AgentSharingGroup != "")
                            {
                                throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name);
                            }
                            if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
                            {
                                throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform);
                            }
                            if (NodeToDo.AgentRequirements != "")
                            {
                                throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name);
                            }
                        }

                        string ProcedureInfix = "";
                        if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
                        {
                            ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString();
                        }

                        bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor;

                        TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode;

                        List <Tuple <string, string> > Parameters = new List <Tuple <string, string> >();

                        Parameters.Add(new Tuple <string, string>("NodeName", NodeToDo.Name));
                        Parameters.Add(new Tuple <string, string>("Sticky", NodeToDo.IsSticky ? "1" : "0"));

                        if (NodeToDo.AgentSharingGroup != "")
                        {
                            Parameters.Add(new Tuple <string, string>("AgentSharingGroup", NodeToDo.AgentSharingGroup));
                        }

                        string Procedure;
                        if (TriggerNodeToDo == null || TriggerNodeToDo.IsTriggered)
                        {
                            if (NodeToDo.IsParallelAgentShareEditor)
                            {
                                Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor";
                            }
                            else
                            {
                                Procedure = "GUBP" + ProcedureInfix + "_UAT_Node";
                                if (!NodeToDo.IsSticky)
                                {
                                    Procedure += "_Parallel";
                                }
                                if (NodeToDo.AgentSharingGroup != "")
                                {
                                    Procedure += "_AgentShare";
                                }
                            }
                            if (NodeToDo.IsSticky && NodeToDo == LastSticky)
                            {
                                Procedure += "_Release";
                            }
                        }
                        else
                        {
                            if (TriggerNodeToDo.RequiresRecursiveWorkflow)
                            {
                                Procedure = "GUBP_UAT_Trigger";                 //here we run a recursive workflow to wait for the trigger
                            }
                            else
                            {
                                Procedure = "GUBP_Hardcoded_Trigger";                                 //here we advance the state in the hardcoded workflow so folks can approve
                            }

                            Parameters.Add(new Tuple <string, string>("TriggerState", TriggerNodeToDo.StateName));
                            Parameters.Add(new Tuple <string, string>("ActionText", TriggerNodeToDo.ActionText));
                            Parameters.Add(new Tuple <string, string>("DescText", TriggerNodeToDo.DescriptionText));

                            if (NodeToDo.RecipientsForFailureEmails.Length > 0)
                            {
                                Parameters.Add(new Tuple <string, string>("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails)));
                            }
                        }

                        string ActualParameterArgs = String.Join(", ", Parameters.Select(x => String.Format("{{actualParameterName => '{0}', value => '{1}'}}", x.Item1, x.Item2)));
                        string Args = String.Format("{0}, subprocedure => '{1}', parallel => '{2}', jobStepName => '{3}', actualParameter => [{4}]", BaseArgs, Procedure, DoParallel? 1 : 0, NodeToDo.Name, ActualParameterArgs);

                        List <BuildNode> UncompletedEcDeps = new List <BuildNode>();
                        {
                            foreach (BuildNode Dep in NodeToDo.AllDirectDependencies)
                            {
                                if (!Dep.IsComplete && OrdereredToDo.Contains(Dep))                                 // if something is already finished, we don't put it into EC
                                {
                                    if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(NodeToDo))
                                    {
                                        throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name);
                                    }
                                    UncompletedEcDeps.Add(Dep);
                                }
                            }
                        }

                        string PreCondition = GetPreConditionForNode(OrdereredToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps);
                        string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath);

                        string MyAgentGroup          = NodeToDo.AgentSharingGroup;
                        bool   bDoNestedJobstep      = false;
                        bool   bDoFirstNestedJobstep = false;

                        string NodeParentPath = ParentPath;
                        if (MyAgentGroup != "")
                        {
                            bDoNestedJobstep = true;
                            NodeParentPath   = ParentPath + "/jobSteps[" + MyAgentGroup + "]";

                            List <BuildNode> MyChain = AgentGroupChains[MyAgentGroup];
                            int MyIndex = MyChain.IndexOf(NodeToDo);
                            if (MyIndex <= 0)
                            {
                                bDoFirstNestedJobstep = bDoNestedJobstep;
                            }
                        }

                        if (bDoNestedJobstep)
                        {
                            if (bDoFirstNestedJobstep)
                            {
                                {
                                    string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}', jobStepName => '{1}', parallel => '1'",
                                                                    ParentPath, MyAgentGroup);
                                    if (!String.IsNullOrEmpty(PreCondition))
                                    {
                                        NestArgs = NestArgs + ", precondition => " + PreCondition;
                                    }
                                    NestArgs = NestArgs + "});";
                                    StepList.Add(NestArgs);
                                }
                                {
                                    string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetPool', subprocedure => 'GUBP{3}_AgentShare_GetPool', parallel => '1', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{4}'}}, {{actualParameterName => 'NodeName', value => '{5}'}}]",
                                                                    ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, MyAgentGroup, NodeToDo.Name);
                                    if (!String.IsNullOrEmpty(PreCondition))
                                    {
                                        NestArgs = NestArgs + ", precondition => " + PreCondition;
                                    }
                                    NestArgs = NestArgs + "});";
                                    StepList.Add(NestArgs);
                                }
                                {
                                    string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetAgent', subprocedure => 'GUBP{3}_AgentShare_GetAgent', parallel => '1', exclusiveMode => 'call', resourceName => '{4}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{5}'}}, {{actualParameterName => 'NodeName', value=> '{6}'}}]",
                                                                    ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix,
                                                                    String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", MyAgentGroup),
                                                                    MyAgentGroup, NodeToDo.Name);
                                    {
                                        NestArgs = NestArgs + ", precondition  => ";
                                        NestArgs = NestArgs + "\"\\$\" . \"[/javascript if(";
                                        NestArgs = NestArgs + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetPool]/status') == 'completed'";
                                        NestArgs = NestArgs + ") true;]\"";
                                    }
                                    NestArgs = NestArgs + "});";
                                    StepList.Add(NestArgs);
                                }
                                {
                                    PreCondition = "\"\\$\" . \"[/javascript if(";
                                    PreCondition = PreCondition + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetAgent]/status') == 'completed'";
                                    PreCondition = PreCondition + ") true;]\"";
                                }
                            }
                            Args = Args.Replace(String.Format("parentPath => '{0}'", ParentPath), String.Format("parentPath => '{0}'", NodeParentPath));
                            Args = Args.Replace("UAT_Node_Parallel_AgentShare", "UAT_Node_Parallel_AgentShare3");
                        }

                        if (!String.IsNullOrEmpty(PreCondition))
                        {
                            Args = Args + ", precondition => " + PreCondition;
                        }
                        if (!String.IsNullOrEmpty(RunCondition))
                        {
                            Args = Args + ", condition => " + RunCondition;
                        }
                #if false
                        // this doesn't work because it includes precondition time
                        if (GUBPNodes[NodeToDo].TimeoutInMinutes() > 0)
                        {
                            Args = Args + String.Format(" --timeLimitUnits minutes --timeLimit {0}", GUBPNodes[NodeToDo].TimeoutInMinutes());
                        }
                #endif
                        if (Sticky && NodeToDo == LastSticky)
                        {
                            Args = Args + ", releaseMode => 'release'";
                        }
                        Args = Args + "});";
                        StepList.Add(Args);

                        if (MyAgentGroup != "" && !bDoNestedJobstep)
                        {
                            List <BuildNode> MyChain = AgentGroupChains[MyAgentGroup];
                            int MyIndex = MyChain.IndexOf(NodeToDo);
                            if (MyIndex == MyChain.Count - 1)
                            {
                                string RelPreCondition = "\"\\$\" . \"[/javascript if(";
                                // this runs "parallel", but we a precondition to serialize it
                                RelPreCondition = RelPreCondition + "getProperty('" + ParentPath + "/jobSteps[" + NodeToDo.Name + "]/status') == 'completed'";
                                RelPreCondition = RelPreCondition + ") true;]\"";
                                // we need to release the resource
                                string RelArgs = String.Format("{0}, subprocedure => 'GUBP_Release_AgentShare', parallel => '1', jobStepName => 'Release_{1}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', valued => '{2}'}}], releaseMode => 'release', precondition => '{3}'",
                                                               BaseArgs, MyAgentGroup, MyAgentGroup, RelPreCondition);
                                StepList.Add(RelArgs);
                            }
                        }
                    }
                }
                WriteECPerl(StepList);
            }
            bool bHasTests = OrdereredToDo.Any(x => x.Node.IsTest());
            RunECTool(String.Format("setProperty \"/myWorkflow/HasTests\" \"{0}\"", bHasTests));
            {
                ECProps.Add("GUBP_LoadedProps=1");
                string BranchDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchDef.properties");
                CommandUtils.WriteAllLines(BranchDefFile, ECProps.ToArray());
                RunECTool(String.Format("setProperty \"/myWorkflow/BranchDefFile\" \"{0}\"", BranchDefFile.Replace("\\", "\\\\")));
            }
            {
                ECProps.Add("GUBP_LoadedJobProps=1");
                string BranchJobDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchJobDef.properties");
                CommandUtils.WriteAllLines(BranchJobDefFile, ECProps.ToArray());
                RunECTool(String.Format("setProperty \"/myJob/BranchJobDefFile\" \"{0}\"", BranchJobDefFile.Replace("\\", "\\\\")));
            }
        }
Beispiel #3
0
        private void WriteJobSteps(List <BuildNode> OrderedToDo, bool bSkipTriggers)
        {
            BuildNode LastSticky   = null;
            bool      HitNonSticky = false;
            bool      bHaveECNodes = false;

            // sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs
            foreach (BuildNode NodeToDo in OrderedToDo)
            {
                if (!NodeToDo.IsComplete)                 // if something is already finished, we don't put it into EC
                {
                    bHaveECNodes = true;
                    if (NodeToDo.IsSticky)
                    {
                        LastSticky = NodeToDo;
                        if (HitNonSticky && !bSkipTriggers)
                        {
                            throw new AutomationException("Sticky and non-sticky jobs did not sort right.");
                        }
                    }
                    else
                    {
                        HitNonSticky = true;
                    }
                }
            }

            List <JobStep> Steps = new List <JobStep>();

            using (TelemetryStopwatch PerlOutputStopwatch = new TelemetryStopwatch("PerlOutput"))
            {
                string ParentPath = Command.ParseParamValue("ParentPath");

                bool bHasNoop = false;
                if (LastSticky == null && bHaveECNodes)
                {
                    // if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource
                    JobStep NoopStep = new JobStep(ParentPath, "Noop", "GUBP_UAT_Node", false, null, null, JobStepReleaseMode.Release);
                    NoopStep.ActualParameters.Add("NodeName", "Noop");
                    NoopStep.ActualParameters.Add("Sticky", "1");
                    Steps.Add(NoopStep);

                    bHasNoop = true;
                }

                Dictionary <string, List <BuildNode> > AgentGroupChains = new Dictionary <string, List <BuildNode> >();
                List <BuildNode> StickyChain = new List <BuildNode>();
                foreach (BuildNode NodeToDo in OrderedToDo)
                {
                    if (!NodeToDo.IsComplete)                     // if something is already finished, we don't put it into EC
                    {
                        string MyAgentGroup = NodeToDo.AgentSharingGroup;
                        if (MyAgentGroup != "")
                        {
                            if (!AgentGroupChains.ContainsKey(MyAgentGroup))
                            {
                                AgentGroupChains.Add(MyAgentGroup, new List <BuildNode> {
                                    NodeToDo
                                });
                            }
                            else
                            {
                                AgentGroupChains[MyAgentGroup].Add(NodeToDo);
                            }
                        }
                    }
                    if (NodeToDo.IsSticky)
                    {
                        if (!StickyChain.Contains(NodeToDo))
                        {
                            StickyChain.Add(NodeToDo);
                        }
                    }
                }
                foreach (BuildNode NodeToDo in OrderedToDo)
                {
                    if (!NodeToDo.IsComplete)                     // if something is already finished, we don't put it into EC
                    {
                        bool Sticky = NodeToDo.IsSticky;
                        if (NodeToDo.IsSticky)
                        {
                            if (NodeToDo.AgentSharingGroup != "")
                            {
                                throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name);
                            }
                            if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
                            {
                                throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform);
                            }
                            if (NodeToDo.AgentRequirements != "")
                            {
                                throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name);
                            }
                        }

                        string ProcedureInfix = "";
                        if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
                        {
                            ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString();
                        }

                        bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor;

                        List <BuildNode> UncompletedEcDeps = new List <BuildNode>();
                        foreach (BuildNode Dep in FindDirectOrderDependencies(NodeToDo))
                        {
                            if (!Dep.IsComplete && OrderedToDo.Contains(Dep))                             // if something is already finished, we don't put it into EC
                            {
                                if (OrderedToDo.IndexOf(Dep) > OrderedToDo.IndexOf(NodeToDo))
                                {
                                    throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name);
                                }
                                UncompletedEcDeps.Add(Dep);
                            }
                        }

                        string PreCondition = GetPreConditionForNode(OrderedToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps);
                        string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath);

                        // Create the job steps for this node
                        TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode;
                        if (TriggerNodeToDo == null)
                        {
                            // Create the jobs to setup the agent sharing group if necessary
                            string NodeParentPath = ParentPath;
                            if (NodeToDo.AgentSharingGroup != "")
                            {
                                NodeParentPath = String.Format("{0}/jobSteps[{1}]", NodeParentPath, NodeToDo.AgentSharingGroup);

                                List <BuildNode> MyChain = AgentGroupChains[NodeToDo.AgentSharingGroup];
                                if (MyChain.IndexOf(NodeToDo) <= 0)
                                {
                                    // Create the parent job step for this group
                                    JobStep ParentStep = new JobStep(ParentPath, NodeToDo.AgentSharingGroup, null, true, PreCondition, null, JobStepReleaseMode.Keep);
                                    Steps.Add(ParentStep);

                                    // Get the resource pool
                                    JobStep GetPoolStep = new JobStep(NodeParentPath, String.Format("{0}_GetPool", NodeToDo.AgentSharingGroup), String.Format("GUBP{0}_AgentShare_GetPool", ProcedureInfix), true, PreCondition, null, JobStepReleaseMode.Keep);
                                    GetPoolStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup);
                                    GetPoolStep.ActualParameters.Add("NodeName", NodeToDo.Name);
                                    Steps.Add(GetPoolStep);

                                    // Get the agent for this sharing group
                                    JobStep GetAgentStep = new JobStep(NodeParentPath, String.Format("{0}_GetAgent", NodeToDo.AgentSharingGroup), String.Format("GUBP{0}_AgentShare_GetAgent", ProcedureInfix), true, GetPoolStep.GetCompletedCondition(), null, JobStepReleaseMode.Keep);
                                    GetAgentStep.Exclusive    = JobStepExclusiveMode.Call;
                                    GetAgentStep.ResourceName = String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", NodeToDo.AgentSharingGroup);
                                    GetAgentStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup);
                                    GetAgentStep.ActualParameters.Add("NodeName", NodeToDo.Name);
                                    Steps.Add(GetAgentStep);

                                    // Set the precondition from this point onwards to be whether the group was set up, since it can't succeed unless the original precondition succeeded
                                    PreCondition = GetAgentStep.GetCompletedCondition();
                                }
                            }

                            // Get the procedure name
                            string Procedure;
                            if (NodeToDo.IsParallelAgentShareEditor)
                            {
                                if (NodeToDo.AgentSharingGroup == "")
                                {
                                    Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor";
                                }
                                else
                                {
                                    Procedure = "GUBP_UAT_Node_Parallel_AgentShare3_Editor";
                                }
                            }
                            else
                            {
                                if (NodeToDo.IsSticky)
                                {
                                    Procedure = "GUBP" + ProcedureInfix + "_UAT_Node";
                                }
                                else if (NodeToDo.AgentSharingGroup == "")
                                {
                                    Procedure = String.Format("GUBP{0}_UAT_Node_Parallel", ProcedureInfix);
                                }
                                else
                                {
                                    Procedure = String.Format("GUBP{0}_UAT_Node_Parallel_AgentShare3", ProcedureInfix);
                                }
                            }
                            if (NodeToDo.IsSticky && NodeToDo == LastSticky)
                            {
                                Procedure += "_Release";
                            }

                            // Build the job step for this node
                            JobStep MainStep = new JobStep(NodeParentPath, NodeToDo.Name, Procedure, DoParallel, PreCondition, RunCondition, (NodeToDo.IsSticky && NodeToDo == LastSticky) ? JobStepReleaseMode.Release : JobStepReleaseMode.Keep);
                            MainStep.ActualParameters.Add("NodeName", NodeToDo.Name);
                            MainStep.ActualParameters.Add("Sticky", NodeToDo.IsSticky ? "1" : "0");
                            if (NodeToDo.AgentSharingGroup != "")
                            {
                                MainStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup);
                            }
                            Steps.Add(MainStep);
                        }
                        else
                        {
                            // Get the procedure name
                            string Procedure;
                            if (TriggerNodeToDo.IsTriggered)
                            {
                                Procedure = String.Format("GUBP{0}_UAT_Node{1}", ProcedureInfix, (NodeToDo == LastSticky) ? "_Release" : "");
                            }
                            else if (TriggerNodeToDo.RequiresRecursiveWorkflow)
                            {
                                Procedure = "GUBP_UAT_Trigger";                                 //here we run a recursive workflow to wait for the trigger
                            }
                            else
                            {
                                Procedure = "GUBP_Hardcoded_Trigger";                                 //here we advance the state in the hardcoded workflow so folks can approve
                            }

                            // Create the job step
                            JobStep TriggerStep = new JobStep(ParentPath, NodeToDo.Name, Procedure, DoParallel, PreCondition, RunCondition, (NodeToDo.IsSticky && NodeToDo == LastSticky) ? JobStepReleaseMode.Release : JobStepReleaseMode.Keep);
                            TriggerStep.ActualParameters.Add("NodeName", NodeToDo.Name);
                            TriggerStep.ActualParameters.Add("Sticky", NodeToDo.IsSticky ? "1" : "0");
                            if (!TriggerNodeToDo.IsTriggered)
                            {
                                TriggerStep.ActualParameters.Add("TriggerState", TriggerNodeToDo.StateName);
                                TriggerStep.ActualParameters.Add("ActionText", TriggerNodeToDo.ActionText);
                                TriggerStep.ActualParameters.Add("DescText", TriggerNodeToDo.DescriptionText);
                                if (NodeToDo.RecipientsForFailureEmails.Length > 0)
                                {
                                    TriggerStep.ActualParameters.Add("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails));
                                }
                            }
                            Steps.Add(TriggerStep);
                        }
                    }
                }
                WriteECPerl(Steps);
            }
        }
Beispiel #4
0
        private void WriteJson(List <BuildNode> OrderedToDo, bool bSkipTriggers)
        {
            using (JsonWriter JsonWriter = new JsonWriter(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "JobSteps.json")))
            {
                JsonWriter.WriteObjectStart();
                JsonWriter.WriteArrayStart("JobSteps");

                string CurrentAgentGroup = "";
                foreach (BuildNode NodeToDo in OrderedToDo)
                {
                    if (!NodeToDo.IsComplete)                     // if something is already finished, we don't put it into EC
                    {
                        // Write the agent group object
                        string AgentGroup = GetAgentGroupOrSticky(NodeToDo);
                        if (AgentGroup != CurrentAgentGroup)
                        {
                            if (CurrentAgentGroup != "")
                            {
                                JsonWriter.WriteArrayEnd();
                                JsonWriter.WriteObjectEnd();
                            }
                            CurrentAgentGroup = AgentGroup;
                            if (CurrentAgentGroup != "")
                            {
                                JsonWriter.WriteObjectStart();
                                JsonWriter.WriteValue("Group", AgentGroup);
                                JsonWriter.WriteValue("AgentPlatform", NodeToDo.AgentPlatform.ToString());
                                JsonWriter.WriteArrayStart("JobSteps");
                            }
                        }

                        // Add this node
                        JsonWriter.WriteObjectStart();
                        JsonWriter.WriteValue("Node", NodeToDo.Name);
                        if (CurrentAgentGroup == "")
                        {
                            JsonWriter.WriteValue("AgentPlatform", NodeToDo.AgentPlatform.ToString());
                        }
                        if (!String.IsNullOrEmpty(NodeToDo.AgentRequirements))
                        {
                            JsonWriter.WriteValue("AgentRequirements", NodeToDo.AgentRequirements);
                        }

                        // Write all the dependencies
                        StringBuilder DependsOnList = new StringBuilder();
                        foreach (BuildNode Dep in FindDirectOrderDependencies(NodeToDo))
                        {
                            if (!Dep.IsComplete && OrderedToDo.Contains(Dep))                             // if something is already finished, we don't put it into EC
                            {
                                if (OrderedToDo.IndexOf(Dep) > OrderedToDo.IndexOf(NodeToDo))
                                {
                                    throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name);
                                }
                                if (DependsOnList.Length > 0)
                                {
                                    DependsOnList.Append(";");
                                }
                                string DepAgentGroup = GetAgentGroupOrSticky(Dep);
                                if (DepAgentGroup != "")
                                {
                                    DependsOnList.AppendFormat("{0}/", DepAgentGroup);
                                }
                                DependsOnList.Append(Dep.Name);
                            }
                        }
                        JsonWriter.WriteValue("DependsOn", DependsOnList.ToString());

                        // Add any trigger-specific settings
                        TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode;
                        if (TriggerNodeToDo != null && !TriggerNodeToDo.IsTriggered)
                        {
                            JsonWriter.WriteValue("TriggerState", TriggerNodeToDo.StateName);
                            JsonWriter.WriteValue("TriggerActionText", TriggerNodeToDo.ActionText);
                            JsonWriter.WriteValue("TriggerDescriptionText", TriggerNodeToDo.DescriptionText);
                            if (NodeToDo.RecipientsForFailureEmails.Length > 0)
                            {
                                JsonWriter.WriteValue("TriggerEmailRecipients", String.Join(" ", NodeToDo.RecipientsForFailureEmails));
                            }
                        }

                        JsonWriter.WriteObjectEnd();
                    }
                }
                if (CurrentAgentGroup != "")
                {
                    JsonWriter.WriteArrayEnd();
                    JsonWriter.WriteObjectEnd();
                }
                JsonWriter.WriteArrayEnd();
                JsonWriter.WriteObjectEnd();
            }
        }
Beispiel #5
0
        public void DoCommanderSetup(IEnumerable <BuildNode> AllNodes, IEnumerable <AggregateNode> AllAggregates, List <BuildNode> OrderedToDo, List <BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, TriggerNode ExplicitTrigger, List <TriggerNode> UnfinishedTriggers)
        {
            // Check there's some nodes in the graph
            if (OrderedToDo.Count == 0)
            {
                throw new AutomationException("No nodes to do!");
            }

            // Make sure everything before the explicit trigger is completed.
            if (ExplicitTrigger != null)
            {
                foreach (BuildNode NodeToDo in OrderedToDo)
                {
                    if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger))                     // if something is already finished, we don't put it into EC
                    {
                        throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name);
                    }
                }
            }

            // Update all the EC properties, and the branch definition files
            WriteProperties(AllNodes, AllAggregates, OrderedToDo, SortedNodes, TimeIndex, TimeQuantum);

            // Write all the job setup
            WriteJobSteps(OrderedToDo, bSkipTriggers);

            // Export it as JSON too
            WriteJson(OrderedToDo, bSkipTriggers);
        }
        public void DoCommanderSetup(IEnumerable<BuildNode> AllNodes, IEnumerable<AggregateNode> AllAggregates, List<BuildNode> OrderedToDo, List<BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, TriggerNode ExplicitTrigger, List<TriggerNode> UnfinishedTriggers, string FakeFail)
		{
			// Check there's some nodes in the graph
			if (OrderedToDo.Count == 0)
			{
				throw new AutomationException("No nodes to do!");
			}

			// Make sure everything before the explicit trigger is completed.
			if (ExplicitTrigger != null)
			{
				foreach (BuildNode NodeToDo in OrderedToDo)
				{
					if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger)) // if something is already finished, we don't put it into EC
					{
						throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name);
					}
				}
			}

			// Update all the EC properties, and the branch definition files
			WriteProperties(AllNodes, AllAggregates, OrderedToDo, SortedNodes, TimeIndex, TimeQuantum);

			// Write all the job setup
			WriteJobSteps(OrderedToDo, bSkipTriggers);
		}
Beispiel #7
0
		public void DoCommanderSetup(IEnumerable<BuildNode> AllNodes, IEnumerable<AggregateNode> AllAggregates, List<BuildNode> OrdereredToDo, List<BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, string CLString, TriggerNode ExplicitTrigger, List<TriggerNode> UnfinishedTriggers, string FakeFail)
		{
			List<AggregateNode> SeparatePromotables = FindPromotables(AllAggregates);
			Dictionary<BuildNode, List<AggregateNode>> DependentPromotions = FindDependentPromotables(AllNodes, SeparatePromotables);

			Dictionary<BuildNode, int> FullNodeListSortKey = GetDisplayOrder(SortedNodes);

			if (OrdereredToDo.Count == 0)
			{
				throw new AutomationException("No nodes to do!");
			}
			List<string> ECProps = new List<string>();
			ECProps.Add(String.Format("TimeIndex={0}", TimeIndex));
			foreach (BuildNode Node in SortedNodes.Where(x => x.FrequencyShift != BuildNode.ExplicitFrequencyShift))
			{
				ECProps.Add(string.Format("AllNodes/{0}={1}", Node.Name, GetNodeForAllNodesProperty(Node, TimeQuantum)));
			}
			foreach (KeyValuePair<BuildNode, int> NodePair in FullNodeListSortKey.Where(x => x.Key.FrequencyShift != BuildNode.ExplicitFrequencyShift))
			{
				ECProps.Add(string.Format("SortKey/{0}={1}", NodePair.Key.Name, NodePair.Value));
			}
			foreach (KeyValuePair<BuildNode, List<AggregateNode>> NodePair in DependentPromotions)
			{
				ECProps.Add(string.Format("DependentPromotions/{0}={1}", NodePair.Key.Name, String.Join(" ", NodePair.Value.Select(x => x.Name))));
			}
			foreach (AggregateNode Node in SeparatePromotables)
			{
				ECProps.Add(string.Format("PossiblePromotables/{0}={1}", Node.Name, ""));
			}
			List<string> ECJobProps = new List<string>();
			if (ExplicitTrigger != null)
			{
				ECJobProps.Add("IsRoot=0");
			}
			else
			{
				ECJobProps.Add("IsRoot=1");
			}

			// here we are just making sure everything before the explicit trigger is completed.
			if (ExplicitTrigger != null)
			{
				foreach (BuildNode NodeToDo in OrdereredToDo)
				{
					if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger)) // if something is already finished, we don't put it into EC
					{
						throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name);
					}
				}
			}

			BuildNode LastSticky = null;
			bool HitNonSticky = false;
			bool bHaveECNodes = false;
			List<string> StepList = new List<string>();
			StepList.Add("use strict;");
			StepList.Add("use diagnostics;");
			StepList.Add("use ElectricCommander();");
			StepList.Add("my $ec = new ElectricCommander;");
			StepList.Add("$ec->setTimeout(600);");
			StepList.Add("my $batch = $ec->newBatch(\"serial\");");
			// sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs
			foreach (BuildNode NodeToDo in OrdereredToDo)
			{
				if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
				{
					bHaveECNodes = true;
					if (NodeToDo.IsSticky)
					{
						LastSticky = NodeToDo;
						if (HitNonSticky && !bSkipTriggers)
						{
							throw new AutomationException("Sticky and non-sticky jobs did not sort right.");
						}
					}
					else
					{
						HitNonSticky = true;
					}
				}
			}

			using(CommandUtils.TelemetryStopwatch PerlOutputStopwatch = new CommandUtils.TelemetryStopwatch("PerlOutput"))
			{
				string ParentPath = Command.ParseParamValue("ParentPath");
				string BaseArgs = String.Format("$batch->createJobStep({{parentPath => '{0}'", ParentPath);

				bool bHasNoop = false;
				if (LastSticky == null && bHaveECNodes)
				{
					// if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource 
					string Args = String.Format("{0}, subprocedure => 'GUBP_UAT_Node', parallel => '0', jobStepName => 'Noop', actualParameter => [{{actualParameterName => 'NodeName', value => 'Noop'}}, {{actualParameterName => 'Sticky', value =>'1' }}], releaseMode => 'release'}});", BaseArgs);
					StepList.Add(Args);
					bHasNoop = true;
				}

				Dictionary<string, List<BuildNode>> AgentGroupChains = new Dictionary<string, List<BuildNode>>();
				List<BuildNode> StickyChain = new List<BuildNode>();
				foreach (BuildNode NodeToDo in OrdereredToDo)
				{
					if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC  
					{
						string MyAgentGroup = NodeToDo.AgentSharingGroup;
						if (MyAgentGroup != "")
						{
							if (!AgentGroupChains.ContainsKey(MyAgentGroup))
							{
								AgentGroupChains.Add(MyAgentGroup, new List<BuildNode> { NodeToDo });
							}
							else
							{
								AgentGroupChains[MyAgentGroup].Add(NodeToDo);
							}
						}
					}
					if (NodeToDo.IsSticky)
					{
						if (!StickyChain.Contains(NodeToDo))
						{
							StickyChain.Add(NodeToDo);
						}
					}
				}
				foreach (BuildNode NodeToDo in OrdereredToDo)
				{
					if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC  
					{
						List<string> NodeProps = GetECPropsForNode(NodeToDo);
						ECProps.AddRange(NodeProps);

						bool Sticky = NodeToDo.IsSticky;
						if(NodeToDo.IsSticky)
						{
							if(NodeToDo.AgentSharingGroup != "")
							{
								throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name);
							}
							if(NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
							{
								throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform);
							}
							if(NodeToDo.AgentRequirements != "")
							{
								throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name);
							}
						}

						string ProcedureInfix = "";
						if(NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
						{
							ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString();
						}

						bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor;
						
						TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode;

						List<Tuple<string, string>> Parameters = new List<Tuple<string,string>>();

						Parameters.Add(new Tuple<string, string>("NodeName", NodeToDo.Name));
						Parameters.Add(new Tuple<string, string>("Sticky", NodeToDo.IsSticky ? "1" : "0"));

						if (NodeToDo.AgentSharingGroup != "")
						{
							Parameters.Add(new Tuple<string, string>("AgentSharingGroup", NodeToDo.AgentSharingGroup));
						}

						string Procedure;
						if(TriggerNodeToDo == null || TriggerNodeToDo.IsTriggered)
						{
							if (NodeToDo.IsParallelAgentShareEditor)
							{
								Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor";
							}
							else
							{
								Procedure = "GUBP" + ProcedureInfix + "_UAT_Node";
								if (!NodeToDo.IsSticky)
								{
									Procedure += "_Parallel";
								}
								if (NodeToDo.AgentSharingGroup != "")
								{
									Procedure += "_AgentShare";
								}
							}
							if (NodeToDo.IsSticky && NodeToDo == LastSticky)
							{
								Procedure += "_Release";
							}
						}
						else
						{
							if(TriggerNodeToDo.RequiresRecursiveWorkflow)
							{
				                Procedure = "GUBP_UAT_Trigger"; //here we run a recursive workflow to wait for the trigger
							}
							else
							{
								Procedure = "GUBP_Hardcoded_Trigger"; //here we advance the state in the hardcoded workflow so folks can approve
							}

							Parameters.Add(new Tuple<string, string>("TriggerState", TriggerNodeToDo.StateName));
							Parameters.Add(new Tuple<string, string>("ActionText", TriggerNodeToDo.ActionText));
							Parameters.Add(new Tuple<string, string>("DescText", TriggerNodeToDo.DescriptionText));

							if (NodeToDo.RecipientsForFailureEmails.Length > 0)
							{
								Parameters.Add(new Tuple<string, string>("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails)));
							}
						}

						string ActualParameterArgs = String.Join(", ", Parameters.Select(x => String.Format("{{actualParameterName => '{0}', value => '{1}'}}", x.Item1, x.Item2)));
						string Args = String.Format("{0}, subprocedure => '{1}', parallel => '{2}', jobStepName => '{3}', actualParameter => [{4}]", BaseArgs, Procedure, DoParallel? 1 : 0, NodeToDo.Name, ActualParameterArgs);
						
						List<BuildNode> UncompletedEcDeps = new List<BuildNode>();
						{
							foreach (BuildNode Dep in NodeToDo.AllDirectDependencies)
							{
								if (!Dep.IsComplete && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC
								{
									if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(NodeToDo))
									{
										throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name);
									}
									UncompletedEcDeps.Add(Dep);
								}
							}
						}

						string PreCondition = GetPreConditionForNode(OrdereredToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps);
						string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath);

						string MyAgentGroup = NodeToDo.AgentSharingGroup;
						bool bDoNestedJobstep = false;
						bool bDoFirstNestedJobstep = false;

						string NodeParentPath = ParentPath;
						if (MyAgentGroup != "")
						{
							bDoNestedJobstep = true;
							NodeParentPath = ParentPath + "/jobSteps[" + MyAgentGroup + "]";

							List<BuildNode> MyChain = AgentGroupChains[MyAgentGroup];
							int MyIndex = MyChain.IndexOf(NodeToDo);
							if (MyIndex <= 0)
							{
								bDoFirstNestedJobstep = bDoNestedJobstep;
							}
						}

						if (bDoNestedJobstep)
						{
							if (bDoFirstNestedJobstep)
							{
								{
									string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}', jobStepName => '{1}', parallel => '1'",
										ParentPath, MyAgentGroup);
									if (!String.IsNullOrEmpty(PreCondition))
									{
										NestArgs = NestArgs + ", precondition => " + PreCondition;
									}
									NestArgs = NestArgs + "});";
									StepList.Add(NestArgs);
								}
								{
									string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetPool', subprocedure => 'GUBP{3}_AgentShare_GetPool', parallel => '1', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{4}'}}, {{actualParameterName => 'NodeName', value => '{5}'}}]",
										ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, MyAgentGroup, NodeToDo.Name);
									if (!String.IsNullOrEmpty(PreCondition))
									{
										NestArgs = NestArgs + ", precondition => " + PreCondition;
									}
									NestArgs = NestArgs + "});";
									StepList.Add(NestArgs);
								}
								{
									string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetAgent', subprocedure => 'GUBP{3}_AgentShare_GetAgent', parallel => '1', exclusiveMode => 'call', resourceName => '{4}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{5}'}}, {{actualParameterName => 'NodeName', value=> '{6}'}}]",
										ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix,
										String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", MyAgentGroup),
										MyAgentGroup, NodeToDo.Name);
									{
										NestArgs = NestArgs + ", precondition  => ";
										NestArgs = NestArgs + "\"\\$\" . \"[/javascript if(";
										NestArgs = NestArgs + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetPool]/status') == 'completed'";
										NestArgs = NestArgs + ") true;]\"";
									}
									NestArgs = NestArgs + "});";
									StepList.Add(NestArgs);
								}
								{
									PreCondition = "\"\\$\" . \"[/javascript if(";
									PreCondition = PreCondition + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetAgent]/status') == 'completed'";
									PreCondition = PreCondition + ") true;]\"";
								}
							}
							Args = Args.Replace(String.Format("parentPath => '{0}'", ParentPath), String.Format("parentPath => '{0}'", NodeParentPath));
							Args = Args.Replace("UAT_Node_Parallel_AgentShare", "UAT_Node_Parallel_AgentShare3");
						}

						if (!String.IsNullOrEmpty(PreCondition))
						{
							Args = Args + ", precondition => " + PreCondition;
						}
						if (!String.IsNullOrEmpty(RunCondition))
						{
							Args = Args + ", condition => " + RunCondition;
						}
		#if false
							// this doesn't work because it includes precondition time
							if (GUBPNodes[NodeToDo].TimeoutInMinutes() > 0)
							{
								Args = Args + String.Format(" --timeLimitUnits minutes --timeLimit {0}", GUBPNodes[NodeToDo].TimeoutInMinutes());
							}
		#endif
						if (Sticky && NodeToDo == LastSticky)
						{
							Args = Args + ", releaseMode => 'release'";
						}
						Args = Args + "});";
						StepList.Add(Args);

						if (MyAgentGroup != "" && !bDoNestedJobstep)
						{
							List<BuildNode> MyChain = AgentGroupChains[MyAgentGroup];
							int MyIndex = MyChain.IndexOf(NodeToDo);
							if (MyIndex == MyChain.Count - 1)
							{
								string RelPreCondition = "\"\\$\" . \"[/javascript if(";
								// this runs "parallel", but we a precondition to serialize it
								RelPreCondition = RelPreCondition + "getProperty('" + ParentPath + "/jobSteps[" + NodeToDo.Name + "]/status') == 'completed'";
								RelPreCondition = RelPreCondition + ") true;]\"";
								// we need to release the resource
								string RelArgs = String.Format("{0}, subprocedure => 'GUBP_Release_AgentShare', parallel => '1', jobStepName => 'Release_{1}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', valued => '{2}'}}], releaseMode => 'release', precondition => '{3}'",
									BaseArgs, MyAgentGroup, MyAgentGroup, RelPreCondition);
								StepList.Add(RelArgs);
							}
						}
					}
				}
				WriteECPerl(StepList);
			}
			bool bHasTests = OrdereredToDo.Any(x => x.Node.IsTest());
			RunECTool(String.Format("setProperty \"/myWorkflow/HasTests\" \"{0}\"", bHasTests));
			{
				ECProps.Add("GUBP_LoadedProps=1");
				string BranchDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchDef.properties");
				CommandUtils.WriteAllLines(BranchDefFile, ECProps.ToArray());
				RunECTool(String.Format("setProperty \"/myWorkflow/BranchDefFile\" \"{0}\"", BranchDefFile.Replace("\\", "\\\\")));
			}
			{
				ECProps.Add("GUBP_LoadedJobProps=1");
				string BranchJobDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchJobDef.properties");
				CommandUtils.WriteAllLines(BranchJobDefFile, ECProps.ToArray());
				RunECTool(String.Format("setProperty \"/myJob/BranchJobDefFile\" \"{0}\"", BranchJobDefFile.Replace("\\", "\\\\")));
			}
		}