private static HashSet <BuildNode> FindDirectOrderDependencies(BuildNode Node) { HashSet <BuildNode> DirectDependencies = new HashSet <BuildNode>(Node.OrderDependencies); foreach (BuildNode OrderDependency in Node.OrderDependencies) { DirectDependencies.ExceptWith(OrderDependency.OrderDependencies); } return(DirectDependencies); }
static string GetJobStepPath(BuildNode Dep) { if (Dep.AgentSharingGroup == "") { return("jobSteps[" + Dep.Name + "]"); } else { return("jobSteps[" + Dep.AgentSharingGroup + "]/jobSteps[" + Dep.Name + "]"); } }
static string GetJobStepPath(BuildNode Dep) { if (Dep.AgentSharingGroup == "") { return "jobSteps[" + Dep.Name + "]"; } else { return "jobSteps[" + Dep.AgentSharingGroup + "]/jobSteps[" + Dep.Name + "]"; } }
static void AddNodeAndDependencies(BuildNode Node, Dictionary <BuildNode, List <BuildNode> > NodeDependencies, HashSet <BuildNode> VisitedNodes, Dictionary <BuildNode, int> SortedNodes) { if (!VisitedNodes.Contains(Node)) { VisitedNodes.Add(Node); foreach (BuildNode NodeDependency in NodeDependencies[Node]) { AddNodeAndDependencies(NodeDependency, NodeDependencies, VisitedNodes, SortedNodes); } SortedNodes.Add(Node, SortedNodes.Count); } }
public void UpdateECProps(BuildNode NodeToDo) { try { CommandUtils.Log("Updating node props for node {0}", NodeToDo.Name); RunECTool(String.Format("setProperty \"/myWorkflow/FailEmails/{0}\" \"{1}\"", NodeToDo.Name, String.Join(" ", NodeToDo.RecipientsForFailureEmails)), true); } catch (Exception Ex) { CommandUtils.Log(System.Diagnostics.TraceEventType.Warning, "Failed to UpdateECProps."); CommandUtils.Log(System.Diagnostics.TraceEventType.Warning, LogUtils.FormatException(Ex)); } }
public void UpdateECBuildTime(BuildNode NodeToDo, double BuildDuration) { try { CommandUtils.Log("Updating duration prop for node {0}", NodeToDo.Name); RunECTool(String.Format("setProperty \"/myWorkflow/NodeDuration/{0}\" \"{1}\"", NodeToDo.Name, BuildDuration.ToString())); RunECTool(String.Format("setProperty \"/myJobStep/NodeDuration\" \"{0}\"", BuildDuration.ToString())); } catch (Exception Ex) { CommandUtils.Log(System.Diagnostics.TraceEventType.Warning, "Failed to UpdateECBuildTime."); CommandUtils.Log(System.Diagnostics.TraceEventType.Warning, LogUtils.FormatException(Ex)); } }
List<string> GetECPropsForNode(BuildNode NodeToDo) { List<string> ECProps = new List<string>(); ECProps.Add("FailEmails/" + NodeToDo.Name + "=" + String.Join(" ", NodeToDo.RecipientsForFailureEmails)); string AgentReq = NodeToDo.AgentRequirements; if(Command.ParseParamValue("AgentOverride") != "" && !NodeToDo.Name.Contains("OnMac")) { AgentReq = Command.ParseParamValue("AgentOverride"); } ECProps.Add(string.Format("AgentRequirementString/{0}={1}", NodeToDo.Name, AgentReq)); ECProps.Add(string.Format("RequiredMemory/{0}={1}", NodeToDo.Name, NodeToDo.AgentMemoryRequirement)); ECProps.Add(string.Format("Timeouts/{0}={1}", NodeToDo.Name, NodeToDo.TimeoutInMinutes)); ECProps.Add(string.Format("JobStepPath/{0}={1}", NodeToDo.Name, GetJobStepPath(NodeToDo))); return ECProps; }
/// <summary> /// Recursively find the controlling triggers for the given node. /// </summary> /// <param name="NodeToDo">Node to find the controlling triggers for</param> static void FindControllingTriggers(BuildNode Node) { if (Node.ControllingTriggers == null) { Node.ControllingTriggers = new TriggerNode[0]; // Find the immediate trigger controlling this one List <TriggerNode> PreviousTriggers = new List <TriggerNode>(); foreach (BuildNode Dependency in Node.OrderDependencies) { FindControllingTriggers(Dependency); TriggerNode PreviousTrigger = Dependency as TriggerNode; if (PreviousTrigger == null && Dependency.ControllingTriggers.Length > 0) { PreviousTrigger = Dependency.ControllingTriggers.Last(); } if (PreviousTrigger != null && !PreviousTriggers.Contains(PreviousTrigger)) { PreviousTriggers.Add(PreviousTrigger); } } // Remove previous triggers from the list that aren't the last in the chain. If there's a trigger chain of X.Y.Z, and a node has dependencies behind all three, the only trigger we care about is Z. PreviousTriggers.RemoveAll(x => PreviousTriggers.Any(y => y.ControllingTriggers.Contains(x))); // We only support one direct controlling trigger at the moment (though it may be in a chain with other triggers) if (PreviousTriggers.Count > 1) { throw new AutomationException("Node {0} has multiple controlling triggers: {1}", Node.Name, String.Join(", ", PreviousTriggers.Select(x => x.Name))); } // Update the list of controlling triggers if (PreviousTriggers.Count == 1) { List <TriggerNode> ControllingTriggers = new List <TriggerNode>(); ControllingTriggers.AddRange(PreviousTriggers[0].ControllingTriggers); ControllingTriggers.Add(PreviousTriggers[0]); Node.ControllingTriggers = ControllingTriggers.ToArray(); } } }
List <string> GetECPropsForNode(BuildNode NodeToDo) { List <string> ECProps = new List <string>(); ECProps.Add("FailEmails/" + NodeToDo.Name + "=" + String.Join(" ", NodeToDo.RecipientsForFailureEmails)); string AgentReq = NodeToDo.AgentRequirements; if (Command.ParseParamValue("AgentOverride") != "" && !NodeToDo.Name.Contains("OnMac")) { AgentReq = Command.ParseParamValue("AgentOverride"); } ECProps.Add(string.Format("AgentRequirementString/{0}={1}", NodeToDo.Name, AgentReq)); ECProps.Add(string.Format("RequiredMemory/{0}={1}", NodeToDo.Name, NodeToDo.AgentMemoryRequirement)); ECProps.Add(string.Format("Timeouts/{0}={1}", NodeToDo.Name, NodeToDo.TimeoutInMinutes)); ECProps.Add(string.Format("JobStepPath/{0}={1}", NodeToDo.Name, GetJobStepPath(NodeToDo))); return(ECProps); }
private static string GetNodeForAllNodesProperty(BuildNode Node, int TimeQuantum) { string Note = Node.ControllingTriggerDotName; if (Note == "") { int Minutes = TimeQuantum << Node.FrequencyShift; if (Minutes == TimeQuantum) { Note = "always"; } else if (Minutes < 60) { Note = String.Format("{0}m", Minutes); } else { Note = String.Format("{0}h{1}m", Minutes / 60, Minutes % 60); } } return(Note); }
private static string GetAgentGroupOrSticky(BuildNode NodeToDo) { string AgentGroup = NodeToDo.AgentSharingGroup; if (NodeToDo.IsSticky) { if (NodeToDo.AgentSharingGroup != "") { throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name); } if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform); } if (NodeToDo.AgentRequirements != "") { throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name); } AgentGroup = "Sticky"; } return(AgentGroup); }
public void UpdateEmailProperties(BuildNode NodeToDo, int LastSucceededCL, string FailedString, string FailCauserEMails, string EMailNote, bool SendSuccessForGreenAfterRed) { RunECTool(String.Format("setProperty \"/myWorkflow/LastGreen/{0}\" \"{1}\"", NodeToDo.Name, LastSucceededCL), true); RunECTool(String.Format("setProperty \"/myWorkflow/LastGreen/{0}\" \"{1}\"", NodeToDo.Name, FailedString), true); RunECTool(String.Format("setProperty \"/myWorkflow/FailCausers/{0}\" \"{1}\"", NodeToDo.Name, FailCauserEMails)); RunECTool(String.Format("setProperty \"/myWorkflow/EmailNotes/{0}\" \"{1}\"", NodeToDo.Name, EMailNote)); { HashSet <string> Emails = new HashSet <string>(NodeToDo.RecipientsForFailureEmails); if (Command.ParseParam("CIS") && !NodeToDo.SendSuccessEmail && NodeToDo.AddSubmittersToFailureEmails) { Emails.UnionWith(FailCauserEMails.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)); } RunECTool(String.Format("setProperty \"/myWorkflow/FailEmails/{0}\" \"{1}\"", NodeToDo.Name, String.Join(" ", Emails))); } if (NodeToDo.SendSuccessEmail || SendSuccessForGreenAfterRed) { RunECTool(String.Format("setProperty \"/myWorkflow/SendSuccessEmail/{0}\" \"{1}\"", NodeToDo.Name, "1")); } else { RunECTool(String.Format("setProperty \"/myWorkflow/SendSuccessEmail/{0}\" \"{1}\"", NodeToDo.Name, "0")); } }
public void SaveStatus(BuildNode NodeToDo, string Suffix, string NodeStoreName, bool bSaveSharedTempStorage, string GameNameIfAny, string JobStepIDForFailure = null) { string Contents = "Just a status record: " + Suffix; if (!String.IsNullOrEmpty(JobStepIDForFailure) && CommandUtils.IsBuildMachine) { try { Contents = RunECTool(String.Format("getProperties --jobStepId {0} --recurse 1", JobStepIDForFailure), true); } catch (Exception Ex) { CommandUtils.Log(System.Diagnostics.TraceEventType.Warning, "Failed to get properties for jobstep to save them."); CommandUtils.Log(System.Diagnostics.TraceEventType.Warning, LogUtils.FormatException(Ex)); } } string RecordOfSuccess = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "Logs", NodeToDo.Name + Suffix + ".log"); CommandUtils.CreateDirectory(Path.GetDirectoryName(RecordOfSuccess)); CommandUtils.WriteAllText(RecordOfSuccess, Contents); TempStorage.StoreToTempStorage(CommandUtils.CmdEnv, NodeStoreName + Suffix, new List <string> { RecordOfSuccess }, !bSaveSharedTempStorage, GameNameIfAny); }
public bool DependsOn(BuildNode Node) { return(OrderDependencies.Contains(Node)); }
public bool DependsOn(BuildNode Node) { return AllIndirectDependencies.Contains(Node); }
private string GetPreConditionForNode(List <BuildNode> OrdereredToDo, string PreconditionParentPath, bool bHasNoop, Dictionary <string, List <BuildNode> > AgentGroupChains, List <BuildNode> StickyChain, BuildNode NodeToDo, List <BuildNode> UncompletedEcDeps) { List <BuildNode> PreConditionUncompletedEcDeps = new List <BuildNode>(); if (NodeToDo.AgentSharingGroup == "") { PreConditionUncompletedEcDeps = new List <BuildNode>(UncompletedEcDeps); } else { List <BuildNode> MyChain = AgentGroupChains[NodeToDo.AgentSharingGroup]; int MyIndex = MyChain.IndexOf(NodeToDo); if (MyIndex > 0) { PreConditionUncompletedEcDeps.Add(MyChain[MyIndex - 1]); } else { // to avoid idle agents (and also EC doesn't actually reserve our agent!), we promote all dependencies to the first one foreach (BuildNode Chain in MyChain) { foreach (BuildNode Dep in Chain.AllDirectDependencies) { if (!Dep.IsComplete && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC { if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(Chain)) { throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", Chain.Name, Dep.Name); } if (!MyChain.Contains(Dep) && !PreConditionUncompletedEcDeps.Contains(Dep)) { PreConditionUncompletedEcDeps.Add(Dep); } } } } } } if (NodeToDo.IsSticky) { List <BuildNode> MyChain = StickyChain; int MyIndex = MyChain.IndexOf(NodeToDo); if (MyIndex > 0) { if (!PreConditionUncompletedEcDeps.Contains(MyChain[MyIndex - 1]) && !MyChain[MyIndex - 1].IsComplete) { PreConditionUncompletedEcDeps.Add(MyChain[MyIndex - 1]); } } else { foreach (BuildNode Dep in NodeToDo.AllDirectDependencies) { if (!Dep.IsComplete && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC { if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(NodeToDo)) { throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name); } if (!MyChain.Contains(Dep) && !PreConditionUncompletedEcDeps.Contains(Dep)) { PreConditionUncompletedEcDeps.Add(Dep); } } } } } List <string> JobStepNames = new List <string>(); if (bHasNoop && PreConditionUncompletedEcDeps.Count == 0) { JobStepNames.Add(PreconditionParentPath + "/jobSteps[Noop]"); } else { JobStepNames.AddRange(PreConditionUncompletedEcDeps.Select(x => GetJobStep(PreconditionParentPath, x))); } string PreCondition = ""; if (JobStepNames.Count > 0) { PreCondition = String.Format("\"\\$\" . \"[/javascript if({0}) true;]\"", String.Join(" && ", JobStepNames.Select(x => String.Format("getProperty('{0}/status') == 'completed'", x)))); } return(PreCondition); }
public void DoCommanderSetup(IEnumerable <BuildNode> AllNodes, IEnumerable <AggregateNode> AllAggregates, List <BuildNode> OrdereredToDo, List <BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, string CLString, TriggerNode ExplicitTrigger, List <TriggerNode> UnfinishedTriggers, string FakeFail) { List <AggregateNode> SeparatePromotables = FindPromotables(AllAggregates); Dictionary <BuildNode, List <AggregateNode> > DependentPromotions = FindDependentPromotables(AllNodes, SeparatePromotables); Dictionary <BuildNode, int> FullNodeListSortKey = GetDisplayOrder(SortedNodes); if (OrdereredToDo.Count == 0) { throw new AutomationException("No nodes to do!"); } List <string> ECProps = new List <string>(); ECProps.Add(String.Format("TimeIndex={0}", TimeIndex)); foreach (BuildNode Node in SortedNodes.Where(x => x.FrequencyShift != BuildNode.ExplicitFrequencyShift)) { ECProps.Add(string.Format("AllNodes/{0}={1}", Node.Name, GetNodeForAllNodesProperty(Node, TimeQuantum))); } foreach (KeyValuePair <BuildNode, int> NodePair in FullNodeListSortKey.Where(x => x.Key.FrequencyShift != BuildNode.ExplicitFrequencyShift)) { ECProps.Add(string.Format("SortKey/{0}={1}", NodePair.Key.Name, NodePair.Value)); } foreach (KeyValuePair <BuildNode, List <AggregateNode> > NodePair in DependentPromotions) { ECProps.Add(string.Format("DependentPromotions/{0}={1}", NodePair.Key.Name, String.Join(" ", NodePair.Value.Select(x => x.Name)))); } foreach (AggregateNode Node in SeparatePromotables) { ECProps.Add(string.Format("PossiblePromotables/{0}={1}", Node.Name, "")); } List <string> ECJobProps = new List <string>(); if (ExplicitTrigger != null) { ECJobProps.Add("IsRoot=0"); } else { ECJobProps.Add("IsRoot=1"); } // here we are just making sure everything before the explicit trigger is completed. if (ExplicitTrigger != null) { foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger)) // if something is already finished, we don't put it into EC { throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name); } } } BuildNode LastSticky = null; bool HitNonSticky = false; bool bHaveECNodes = false; List <string> StepList = new List <string>(); StepList.Add("use strict;"); StepList.Add("use diagnostics;"); StepList.Add("use ElectricCommander();"); StepList.Add("my $ec = new ElectricCommander;"); StepList.Add("$ec->setTimeout(600);"); StepList.Add("my $batch = $ec->newBatch(\"serial\");"); // sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { bHaveECNodes = true; if (NodeToDo.IsSticky) { LastSticky = NodeToDo; if (HitNonSticky && !bSkipTriggers) { throw new AutomationException("Sticky and non-sticky jobs did not sort right."); } } else { HitNonSticky = true; } } } using (CommandUtils.TelemetryStopwatch PerlOutputStopwatch = new CommandUtils.TelemetryStopwatch("PerlOutput")) { string ParentPath = Command.ParseParamValue("ParentPath"); string BaseArgs = String.Format("$batch->createJobStep({{parentPath => '{0}'", ParentPath); bool bHasNoop = false; if (LastSticky == null && bHaveECNodes) { // if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource string Args = String.Format("{0}, subprocedure => 'GUBP_UAT_Node', parallel => '0', jobStepName => 'Noop', actualParameter => [{{actualParameterName => 'NodeName', value => 'Noop'}}, {{actualParameterName => 'Sticky', value =>'1' }}], releaseMode => 'release'}});", BaseArgs); StepList.Add(Args); bHasNoop = true; } Dictionary <string, List <BuildNode> > AgentGroupChains = new Dictionary <string, List <BuildNode> >(); List <BuildNode> StickyChain = new List <BuildNode>(); foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { string MyAgentGroup = NodeToDo.AgentSharingGroup; if (MyAgentGroup != "") { if (!AgentGroupChains.ContainsKey(MyAgentGroup)) { AgentGroupChains.Add(MyAgentGroup, new List <BuildNode> { NodeToDo }); } else { AgentGroupChains[MyAgentGroup].Add(NodeToDo); } } } if (NodeToDo.IsSticky) { if (!StickyChain.Contains(NodeToDo)) { StickyChain.Add(NodeToDo); } } } foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { List <string> NodeProps = GetECPropsForNode(NodeToDo); ECProps.AddRange(NodeProps); bool Sticky = NodeToDo.IsSticky; if (NodeToDo.IsSticky) { if (NodeToDo.AgentSharingGroup != "") { throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name); } if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform); } if (NodeToDo.AgentRequirements != "") { throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name); } } string ProcedureInfix = ""; if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString(); } bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor; TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode; List <Tuple <string, string> > Parameters = new List <Tuple <string, string> >(); Parameters.Add(new Tuple <string, string>("NodeName", NodeToDo.Name)); Parameters.Add(new Tuple <string, string>("Sticky", NodeToDo.IsSticky ? "1" : "0")); if (NodeToDo.AgentSharingGroup != "") { Parameters.Add(new Tuple <string, string>("AgentSharingGroup", NodeToDo.AgentSharingGroup)); } string Procedure; if (TriggerNodeToDo == null || TriggerNodeToDo.IsTriggered) { if (NodeToDo.IsParallelAgentShareEditor) { Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor"; } else { Procedure = "GUBP" + ProcedureInfix + "_UAT_Node"; if (!NodeToDo.IsSticky) { Procedure += "_Parallel"; } if (NodeToDo.AgentSharingGroup != "") { Procedure += "_AgentShare"; } } if (NodeToDo.IsSticky && NodeToDo == LastSticky) { Procedure += "_Release"; } } else { if (TriggerNodeToDo.RequiresRecursiveWorkflow) { Procedure = "GUBP_UAT_Trigger"; //here we run a recursive workflow to wait for the trigger } else { Procedure = "GUBP_Hardcoded_Trigger"; //here we advance the state in the hardcoded workflow so folks can approve } Parameters.Add(new Tuple <string, string>("TriggerState", TriggerNodeToDo.StateName)); Parameters.Add(new Tuple <string, string>("ActionText", TriggerNodeToDo.ActionText)); Parameters.Add(new Tuple <string, string>("DescText", TriggerNodeToDo.DescriptionText)); if (NodeToDo.RecipientsForFailureEmails.Length > 0) { Parameters.Add(new Tuple <string, string>("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails))); } } string ActualParameterArgs = String.Join(", ", Parameters.Select(x => String.Format("{{actualParameterName => '{0}', value => '{1}'}}", x.Item1, x.Item2))); string Args = String.Format("{0}, subprocedure => '{1}', parallel => '{2}', jobStepName => '{3}', actualParameter => [{4}]", BaseArgs, Procedure, DoParallel? 1 : 0, NodeToDo.Name, ActualParameterArgs); List <BuildNode> UncompletedEcDeps = new List <BuildNode>(); { foreach (BuildNode Dep in NodeToDo.AllDirectDependencies) { if (!Dep.IsComplete && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC { if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(NodeToDo)) { throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name); } UncompletedEcDeps.Add(Dep); } } } string PreCondition = GetPreConditionForNode(OrdereredToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps); string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath); string MyAgentGroup = NodeToDo.AgentSharingGroup; bool bDoNestedJobstep = false; bool bDoFirstNestedJobstep = false; string NodeParentPath = ParentPath; if (MyAgentGroup != "") { bDoNestedJobstep = true; NodeParentPath = ParentPath + "/jobSteps[" + MyAgentGroup + "]"; List <BuildNode> MyChain = AgentGroupChains[MyAgentGroup]; int MyIndex = MyChain.IndexOf(NodeToDo); if (MyIndex <= 0) { bDoFirstNestedJobstep = bDoNestedJobstep; } } if (bDoNestedJobstep) { if (bDoFirstNestedJobstep) { { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}', jobStepName => '{1}', parallel => '1'", ParentPath, MyAgentGroup); if (!String.IsNullOrEmpty(PreCondition)) { NestArgs = NestArgs + ", precondition => " + PreCondition; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetPool', subprocedure => 'GUBP{3}_AgentShare_GetPool', parallel => '1', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{4}'}}, {{actualParameterName => 'NodeName', value => '{5}'}}]", ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, MyAgentGroup, NodeToDo.Name); if (!String.IsNullOrEmpty(PreCondition)) { NestArgs = NestArgs + ", precondition => " + PreCondition; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetAgent', subprocedure => 'GUBP{3}_AgentShare_GetAgent', parallel => '1', exclusiveMode => 'call', resourceName => '{4}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{5}'}}, {{actualParameterName => 'NodeName', value=> '{6}'}}]", ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", MyAgentGroup), MyAgentGroup, NodeToDo.Name); { NestArgs = NestArgs + ", precondition => "; NestArgs = NestArgs + "\"\\$\" . \"[/javascript if("; NestArgs = NestArgs + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetPool]/status') == 'completed'"; NestArgs = NestArgs + ") true;]\""; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { PreCondition = "\"\\$\" . \"[/javascript if("; PreCondition = PreCondition + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetAgent]/status') == 'completed'"; PreCondition = PreCondition + ") true;]\""; } } Args = Args.Replace(String.Format("parentPath => '{0}'", ParentPath), String.Format("parentPath => '{0}'", NodeParentPath)); Args = Args.Replace("UAT_Node_Parallel_AgentShare", "UAT_Node_Parallel_AgentShare3"); } if (!String.IsNullOrEmpty(PreCondition)) { Args = Args + ", precondition => " + PreCondition; } if (!String.IsNullOrEmpty(RunCondition)) { Args = Args + ", condition => " + RunCondition; } #if false // this doesn't work because it includes precondition time if (GUBPNodes[NodeToDo].TimeoutInMinutes() > 0) { Args = Args + String.Format(" --timeLimitUnits minutes --timeLimit {0}", GUBPNodes[NodeToDo].TimeoutInMinutes()); } #endif if (Sticky && NodeToDo == LastSticky) { Args = Args + ", releaseMode => 'release'"; } Args = Args + "});"; StepList.Add(Args); if (MyAgentGroup != "" && !bDoNestedJobstep) { List <BuildNode> MyChain = AgentGroupChains[MyAgentGroup]; int MyIndex = MyChain.IndexOf(NodeToDo); if (MyIndex == MyChain.Count - 1) { string RelPreCondition = "\"\\$\" . \"[/javascript if("; // this runs "parallel", but we a precondition to serialize it RelPreCondition = RelPreCondition + "getProperty('" + ParentPath + "/jobSteps[" + NodeToDo.Name + "]/status') == 'completed'"; RelPreCondition = RelPreCondition + ") true;]\""; // we need to release the resource string RelArgs = String.Format("{0}, subprocedure => 'GUBP_Release_AgentShare', parallel => '1', jobStepName => 'Release_{1}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', valued => '{2}'}}], releaseMode => 'release', precondition => '{3}'", BaseArgs, MyAgentGroup, MyAgentGroup, RelPreCondition); StepList.Add(RelArgs); } } } } WriteECPerl(StepList); } bool bHasTests = OrdereredToDo.Any(x => x.Node.IsTest()); RunECTool(String.Format("setProperty \"/myWorkflow/HasTests\" \"{0}\"", bHasTests)); { ECProps.Add("GUBP_LoadedProps=1"); string BranchDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchDef.properties"); CommandUtils.WriteAllLines(BranchDefFile, ECProps.ToArray()); RunECTool(String.Format("setProperty \"/myWorkflow/BranchDefFile\" \"{0}\"", BranchDefFile.Replace("\\", "\\\\"))); } { ECProps.Add("GUBP_LoadedJobProps=1"); string BranchJobDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchJobDef.properties"); CommandUtils.WriteAllLines(BranchJobDefFile, ECProps.ToArray()); RunECTool(String.Format("setProperty \"/myJob/BranchJobDefFile\" \"{0}\"", BranchJobDefFile.Replace("\\", "\\\\"))); } }
private string GetPreConditionForNode(List<BuildNode> OrderedToDo, string PreconditionParentPath, bool bHasNoop, Dictionary<string, List<BuildNode>> AgentGroupChains, List<BuildNode> StickyChain, BuildNode NodeToDo, List<BuildNode> UncompletedEcDeps) { List<BuildNode> PreConditionUncompletedEcDeps = new List<BuildNode>(); if(NodeToDo.AgentSharingGroup == "") { PreConditionUncompletedEcDeps = new List<BuildNode>(UncompletedEcDeps); } else { List<BuildNode> MyChain = AgentGroupChains[NodeToDo.AgentSharingGroup]; int MyIndex = MyChain.IndexOf(NodeToDo); if (MyIndex > 0) { PreConditionUncompletedEcDeps.Add(MyChain[MyIndex - 1]); } else { // to avoid idle agents (and also EC doesn't actually reserve our agent!), we promote all dependencies to the first one foreach (BuildNode Chain in MyChain) { foreach (BuildNode Dep in FindDirectOrderDependencies(Chain)) { if (!Dep.IsComplete && OrderedToDo.Contains(Dep)) // if something is already finished, we don't put it into EC { if (OrderedToDo.IndexOf(Dep) > OrderedToDo.IndexOf(Chain)) { throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", Chain.Name, Dep.Name); } if (!MyChain.Contains(Dep) && !PreConditionUncompletedEcDeps.Contains(Dep)) { PreConditionUncompletedEcDeps.Add(Dep); } } } } } } if (NodeToDo.IsSticky) { List<BuildNode> MyChain = StickyChain; int MyIndex = MyChain.IndexOf(NodeToDo); if (MyIndex > 0) { if (!PreConditionUncompletedEcDeps.Contains(MyChain[MyIndex - 1]) && !MyChain[MyIndex - 1].IsComplete) { PreConditionUncompletedEcDeps.Add(MyChain[MyIndex - 1]); } } else { foreach (BuildNode Dep in FindDirectOrderDependencies(NodeToDo)) { if (!Dep.IsComplete && OrderedToDo.Contains(Dep)) // if something is already finished, we don't put it into EC { if (OrderedToDo.IndexOf(Dep) > OrderedToDo.IndexOf(NodeToDo)) { throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name); } if (!MyChain.Contains(Dep) && !PreConditionUncompletedEcDeps.Contains(Dep)) { PreConditionUncompletedEcDeps.Add(Dep); } } } } } List<string> JobStepNames = new List<string>(); if (bHasNoop && PreConditionUncompletedEcDeps.Count == 0) { JobStepNames.Add(PreconditionParentPath + "/jobSteps[Noop]"); } else { JobStepNames.AddRange(PreConditionUncompletedEcDeps.Select(x => GetJobStep(PreconditionParentPath, x))); } string PreCondition = ""; if (JobStepNames.Count > 0) { PreCondition = String.Format("\"\\$\" . \"[/javascript if({0}) true;]\"", String.Join(" && ", JobStepNames.Select(x => String.Format("getProperty('{0}/status') == 'completed'", x)))); } return PreCondition; }
static string GetJobStep(string ParentPath, BuildNode Dep) { return ParentPath + "/" + GetJobStepPath(Dep); }
public void UpdateECProps(BuildNode NodeToDo) { try { CommandUtils.LogConsole("Updating node props for node {0}", NodeToDo.Name); RunECTool(String.Format("setProperty \"/myWorkflow/FailEmails/{0}\" \"{1}\"", NodeToDo.Name, String.Join(" ", NodeToDo.RecipientsForFailureEmails)), true); } catch (Exception Ex) { CommandUtils.LogWarning("Failed to UpdateECProps."); CommandUtils.LogWarning(LogUtils.FormatException(Ex)); } }
static string GetJobStep(string ParentPath, BuildNode Dep) { return(ParentPath + "/" + GetJobStepPath(Dep)); }
public void SaveStatus(BuildNode NodeToDo, string Suffix, string NodeStoreName, bool bSaveSharedTempStorage, string GameNameIfAny, string JobStepIDForFailure = null) { string Contents = "Just a status record: " + Suffix; if (!String.IsNullOrEmpty(JobStepIDForFailure) && CommandUtils.IsBuildMachine) { try { Contents = RunECTool(String.Format("getProperties --jobStepId {0} --recurse 1", JobStepIDForFailure), true); } catch (Exception Ex) { CommandUtils.Log(System.Diagnostics.TraceEventType.Warning, "Failed to get properties for jobstep to save them."); CommandUtils.Log(System.Diagnostics.TraceEventType.Warning, LogUtils.FormatException(Ex)); } } string RecordOfSuccess = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "Logs", NodeToDo.Name + Suffix +".log"); CommandUtils.CreateDirectory(Path.GetDirectoryName(RecordOfSuccess)); CommandUtils.WriteAllText(RecordOfSuccess, Contents); TempStorage.StoreToTempStorage(CommandUtils.CmdEnv, NodeStoreName + Suffix, new List<string> { RecordOfSuccess }, !bSaveSharedTempStorage, GameNameIfAny); }
public void UpdateECBuildTime(BuildNode NodeToDo, double BuildDuration) { try { CommandUtils.LogConsole("Updating duration prop for node {0}", NodeToDo.Name); RunECTool(String.Format("setProperty \"/myWorkflow/NodeDuration/{0}\" \"{1}\"", NodeToDo.Name, BuildDuration.ToString())); RunECTool(String.Format("setProperty \"/myJobStep/NodeDuration\" \"{0}\"", BuildDuration.ToString())); } catch (Exception Ex) { CommandUtils.LogWarning("Failed to UpdateECBuildTime."); CommandUtils.LogWarning(LogUtils.FormatException(Ex)); } }
static void AddNodeAndDependencies(BuildNode Node, Dictionary<BuildNode, List<BuildNode>> NodeDependencies, HashSet<BuildNode> VisitedNodes, Dictionary<BuildNode, int> SortedNodes) { if(!VisitedNodes.Contains(Node)) { VisitedNodes.Add(Node); foreach (BuildNode NodeDependency in NodeDependencies[Node]) { AddNodeAndDependencies(NodeDependency, NodeDependencies, VisitedNodes, SortedNodes); } SortedNodes.Add(Node, SortedNodes.Count); } }
private static string GetNodeForAllNodesProperty(BuildNode Node, int TimeQuantum) { string Note = Node.ControllingTriggerDotName; if (Note == "") { int Minutes = TimeQuantum << Node.FrequencyShift; if (Minutes == TimeQuantum) { Note = "always"; } else if (Minutes < 60) { Note = String.Format("{0}m", Minutes); } else { Note = String.Format("{0}h{1}m", Minutes / 60, Minutes % 60); } } return Note; }
private static HashSet<BuildNode> FindDirectOrderDependencies(BuildNode Node) { HashSet<BuildNode> DirectDependencies = new HashSet<BuildNode>(Node.OrderDependencies); foreach(BuildNode OrderDependency in Node.OrderDependencies) { DirectDependencies.ExceptWith(OrderDependency.OrderDependencies); } return DirectDependencies; }
public bool DependsOn(BuildNode Node) { return(AllIndirectDependencies.Contains(Node)); }
public void UpdateEmailProperties(BuildNode NodeToDo, int LastSucceededCL, string FailedString, string FailCauserEMails, string EMailNote, bool SendSuccessForGreenAfterRed) { RunECTool(String.Format("setProperty \"/myWorkflow/LastGreen/{0}\" \"{1}\"", NodeToDo.Name, LastSucceededCL), true); RunECTool(String.Format("setProperty \"/myWorkflow/LastGreen/{0}\" \"{1}\"", NodeToDo.Name, FailedString), true); RunECTool(String.Format("setProperty \"/myWorkflow/FailCausers/{0}\" \"{1}\"", NodeToDo.Name, FailCauserEMails)); RunECTool(String.Format("setProperty \"/myWorkflow/EmailNotes/{0}\" \"{1}\"", NodeToDo.Name, EMailNote)); { HashSet<string> Emails = new HashSet<string>(NodeToDo.RecipientsForFailureEmails); if (Command.ParseParam("CIS") && !NodeToDo.SendSuccessEmail && NodeToDo.AddSubmittersToFailureEmails) { Emails.UnionWith(FailCauserEMails.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)); } RunECTool(String.Format("setProperty \"/myWorkflow/FailEmails/{0}\" \"{1}\"", NodeToDo.Name, String.Join(" ", Emails))); } if (NodeToDo.SendSuccessEmail || SendSuccessForGreenAfterRed) { RunECTool(String.Format("setProperty \"/myWorkflow/SendSuccessEmail/{0}\" \"{1}\"", NodeToDo.Name, "1")); } else { RunECTool(String.Format("setProperty \"/myWorkflow/SendSuccessEmail/{0}\" \"{1}\"", NodeToDo.Name, "0")); } }
private void WriteJobSteps(List <BuildNode> OrderedToDo, bool bSkipTriggers) { BuildNode LastSticky = null; bool HitNonSticky = false; bool bHaveECNodes = false; // sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs foreach (BuildNode NodeToDo in OrderedToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { bHaveECNodes = true; if (NodeToDo.IsSticky) { LastSticky = NodeToDo; if (HitNonSticky && !bSkipTriggers) { throw new AutomationException("Sticky and non-sticky jobs did not sort right."); } } else { HitNonSticky = true; } } } List <JobStep> Steps = new List <JobStep>(); using (TelemetryStopwatch PerlOutputStopwatch = new TelemetryStopwatch("PerlOutput")) { string ParentPath = Command.ParseParamValue("ParentPath"); bool bHasNoop = false; if (LastSticky == null && bHaveECNodes) { // if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource JobStep NoopStep = new JobStep(ParentPath, "Noop", "GUBP_UAT_Node", false, null, null, JobStepReleaseMode.Release); NoopStep.ActualParameters.Add("NodeName", "Noop"); NoopStep.ActualParameters.Add("Sticky", "1"); Steps.Add(NoopStep); bHasNoop = true; } Dictionary <string, List <BuildNode> > AgentGroupChains = new Dictionary <string, List <BuildNode> >(); List <BuildNode> StickyChain = new List <BuildNode>(); foreach (BuildNode NodeToDo in OrderedToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { string MyAgentGroup = NodeToDo.AgentSharingGroup; if (MyAgentGroup != "") { if (!AgentGroupChains.ContainsKey(MyAgentGroup)) { AgentGroupChains.Add(MyAgentGroup, new List <BuildNode> { NodeToDo }); } else { AgentGroupChains[MyAgentGroup].Add(NodeToDo); } } } if (NodeToDo.IsSticky) { if (!StickyChain.Contains(NodeToDo)) { StickyChain.Add(NodeToDo); } } } foreach (BuildNode NodeToDo in OrderedToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { bool Sticky = NodeToDo.IsSticky; if (NodeToDo.IsSticky) { if (NodeToDo.AgentSharingGroup != "") { throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name); } if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform); } if (NodeToDo.AgentRequirements != "") { throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name); } } string ProcedureInfix = ""; if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString(); } bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor; List <BuildNode> UncompletedEcDeps = new List <BuildNode>(); foreach (BuildNode Dep in FindDirectOrderDependencies(NodeToDo)) { if (!Dep.IsComplete && OrderedToDo.Contains(Dep)) // if something is already finished, we don't put it into EC { if (OrderedToDo.IndexOf(Dep) > OrderedToDo.IndexOf(NodeToDo)) { throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name); } UncompletedEcDeps.Add(Dep); } } string PreCondition = GetPreConditionForNode(OrderedToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps); string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath); // Create the job steps for this node TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode; if (TriggerNodeToDo == null) { // Create the jobs to setup the agent sharing group if necessary string NodeParentPath = ParentPath; if (NodeToDo.AgentSharingGroup != "") { NodeParentPath = String.Format("{0}/jobSteps[{1}]", NodeParentPath, NodeToDo.AgentSharingGroup); List <BuildNode> MyChain = AgentGroupChains[NodeToDo.AgentSharingGroup]; if (MyChain.IndexOf(NodeToDo) <= 0) { // Create the parent job step for this group JobStep ParentStep = new JobStep(ParentPath, NodeToDo.AgentSharingGroup, null, true, PreCondition, null, JobStepReleaseMode.Keep); Steps.Add(ParentStep); // Get the resource pool JobStep GetPoolStep = new JobStep(NodeParentPath, String.Format("{0}_GetPool", NodeToDo.AgentSharingGroup), String.Format("GUBP{0}_AgentShare_GetPool", ProcedureInfix), true, PreCondition, null, JobStepReleaseMode.Keep); GetPoolStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup); GetPoolStep.ActualParameters.Add("NodeName", NodeToDo.Name); Steps.Add(GetPoolStep); // Get the agent for this sharing group JobStep GetAgentStep = new JobStep(NodeParentPath, String.Format("{0}_GetAgent", NodeToDo.AgentSharingGroup), String.Format("GUBP{0}_AgentShare_GetAgent", ProcedureInfix), true, GetPoolStep.GetCompletedCondition(), null, JobStepReleaseMode.Keep); GetAgentStep.Exclusive = JobStepExclusiveMode.Call; GetAgentStep.ResourceName = String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", NodeToDo.AgentSharingGroup); GetAgentStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup); GetAgentStep.ActualParameters.Add("NodeName", NodeToDo.Name); Steps.Add(GetAgentStep); // Set the precondition from this point onwards to be whether the group was set up, since it can't succeed unless the original precondition succeeded PreCondition = GetAgentStep.GetCompletedCondition(); } } // Get the procedure name string Procedure; if (NodeToDo.IsParallelAgentShareEditor) { if (NodeToDo.AgentSharingGroup == "") { Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor"; } else { Procedure = "GUBP_UAT_Node_Parallel_AgentShare3_Editor"; } } else { if (NodeToDo.IsSticky) { Procedure = "GUBP" + ProcedureInfix + "_UAT_Node"; } else if (NodeToDo.AgentSharingGroup == "") { Procedure = String.Format("GUBP{0}_UAT_Node_Parallel", ProcedureInfix); } else { Procedure = String.Format("GUBP{0}_UAT_Node_Parallel_AgentShare3", ProcedureInfix); } } if (NodeToDo.IsSticky && NodeToDo == LastSticky) { Procedure += "_Release"; } // Build the job step for this node JobStep MainStep = new JobStep(NodeParentPath, NodeToDo.Name, Procedure, DoParallel, PreCondition, RunCondition, (NodeToDo.IsSticky && NodeToDo == LastSticky) ? JobStepReleaseMode.Release : JobStepReleaseMode.Keep); MainStep.ActualParameters.Add("NodeName", NodeToDo.Name); MainStep.ActualParameters.Add("Sticky", NodeToDo.IsSticky ? "1" : "0"); if (NodeToDo.AgentSharingGroup != "") { MainStep.ActualParameters.Add("AgentSharingGroup", NodeToDo.AgentSharingGroup); } Steps.Add(MainStep); } else { // Get the procedure name string Procedure; if (TriggerNodeToDo.IsTriggered) { Procedure = String.Format("GUBP{0}_UAT_Node{1}", ProcedureInfix, (NodeToDo == LastSticky) ? "_Release" : ""); } else if (TriggerNodeToDo.RequiresRecursiveWorkflow) { Procedure = "GUBP_UAT_Trigger"; //here we run a recursive workflow to wait for the trigger } else { Procedure = "GUBP_Hardcoded_Trigger"; //here we advance the state in the hardcoded workflow so folks can approve } // Create the job step JobStep TriggerStep = new JobStep(ParentPath, NodeToDo.Name, Procedure, DoParallel, PreCondition, RunCondition, (NodeToDo.IsSticky && NodeToDo == LastSticky) ? JobStepReleaseMode.Release : JobStepReleaseMode.Keep); TriggerStep.ActualParameters.Add("NodeName", NodeToDo.Name); TriggerStep.ActualParameters.Add("Sticky", NodeToDo.IsSticky ? "1" : "0"); if (!TriggerNodeToDo.IsTriggered) { TriggerStep.ActualParameters.Add("TriggerState", TriggerNodeToDo.StateName); TriggerStep.ActualParameters.Add("ActionText", TriggerNodeToDo.ActionText); TriggerStep.ActualParameters.Add("DescText", TriggerNodeToDo.DescriptionText); if (NodeToDo.RecipientsForFailureEmails.Length > 0) { TriggerStep.ActualParameters.Add("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails)); } } Steps.Add(TriggerStep); } } } WriteECPerl(Steps); } }
public bool DependsOn(BuildNode Node) { return OrderDependencies.Contains(Node); }