public BranchInfo(List <UnrealTargetPlatform> InHostPlatforms) { BaseEngineProject = new BranchUProject(); var AllProjects = UnrealBuildTool.UProjectInfo.FilterGameProjects(false, null); using (CommandUtils.TelemetryStopwatch SortProjectsStopwatch = new CommandUtils.TelemetryStopwatch("SortProjects")) { foreach (var InfoEntry in AllProjects) { var UProject = new BranchUProject(InfoEntry); if (UProject.Properties.bIsCodeBasedProject) { CodeProjects.Add(UProject); } else { NonCodeProjects.Add(UProject); // the base project uses BlankProject if it really needs a .uproject file if (String.IsNullOrEmpty(BaseEngineProject.FilePath) && UProject.GameName == "BlankProject") { BaseEngineProject.FilePath = UProject.FilePath; } } } } /* if (String.IsNullOrEmpty(BaseEngineProject.FilePath)) * { * throw new AutomationException("All branches must have the blank project /Samples/Sandbox/BlankProject"); * }*/ using (CommandUtils.TelemetryStopwatch ProjectDumpStopwatch = new CommandUtils.TelemetryStopwatch("Project Dump")) { CommandUtils.Log(" Base Engine:"); BaseEngineProject.Dump(InHostPlatforms); CommandUtils.Log(" {0} Code projects:", CodeProjects.Count); foreach (var Proj in CodeProjects) { Proj.Dump(InHostPlatforms); } CommandUtils.Log(" {0} Non-Code projects:", CodeProjects.Count); foreach (var Proj in NonCodeProjects) { Proj.Dump(InHostPlatforms); } } }
public BranchInfo(List<UnrealTargetPlatform> InHostPlatforms) { BaseEngineProject = new BranchUProject(); var AllProjects = UnrealBuildTool.UProjectInfo.FilterGameProjects(false, null); using(CommandUtils.TelemetryStopwatch SortProjectsStopwatch = new CommandUtils.TelemetryStopwatch("SortProjects")) { foreach (var InfoEntry in AllProjects) { var UProject = new BranchUProject(InfoEntry); if (UProject.Properties.bIsCodeBasedProject) { CodeProjects.Add(UProject); } else { NonCodeProjects.Add(UProject); // the base project uses BlankProject if it really needs a .uproject file if (String.IsNullOrEmpty(BaseEngineProject.FilePath) && UProject.GameName == "BlankProject") { BaseEngineProject.FilePath = UProject.FilePath; } } } } /* if (String.IsNullOrEmpty(BaseEngineProject.FilePath)) { throw new AutomationException("All branches must have the blank project /Samples/Sandbox/BlankProject"); }*/ using(CommandUtils.TelemetryStopwatch ProjectDumpStopwatch = new CommandUtils.TelemetryStopwatch("Project Dump")) { CommandUtils.Log(" Base Engine:"); BaseEngineProject.Dump(InHostPlatforms); CommandUtils.Log(" {0} Code projects:", CodeProjects.Count); foreach (var Proj in CodeProjects) { Proj.Dump(InHostPlatforms); } CommandUtils.Log(" {0} Non-Code projects:", CodeProjects.Count); foreach (var Proj in NonCodeProjects) { Proj.Dump(InHostPlatforms); } } }
public void DoCommanderSetup(IEnumerable <BuildNode> AllNodes, IEnumerable <AggregateNode> AllAggregates, List <BuildNode> OrdereredToDo, List <BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, string CLString, TriggerNode ExplicitTrigger, List <TriggerNode> UnfinishedTriggers, string FakeFail) { List <AggregateNode> SeparatePromotables = FindPromotables(AllAggregates); Dictionary <BuildNode, List <AggregateNode> > DependentPromotions = FindDependentPromotables(AllNodes, SeparatePromotables); Dictionary <BuildNode, int> FullNodeListSortKey = GetDisplayOrder(SortedNodes); if (OrdereredToDo.Count == 0) { throw new AutomationException("No nodes to do!"); } List <string> ECProps = new List <string>(); ECProps.Add(String.Format("TimeIndex={0}", TimeIndex)); foreach (BuildNode Node in SortedNodes.Where(x => x.FrequencyShift != BuildNode.ExplicitFrequencyShift)) { ECProps.Add(string.Format("AllNodes/{0}={1}", Node.Name, GetNodeForAllNodesProperty(Node, TimeQuantum))); } foreach (KeyValuePair <BuildNode, int> NodePair in FullNodeListSortKey.Where(x => x.Key.FrequencyShift != BuildNode.ExplicitFrequencyShift)) { ECProps.Add(string.Format("SortKey/{0}={1}", NodePair.Key.Name, NodePair.Value)); } foreach (KeyValuePair <BuildNode, List <AggregateNode> > NodePair in DependentPromotions) { ECProps.Add(string.Format("DependentPromotions/{0}={1}", NodePair.Key.Name, String.Join(" ", NodePair.Value.Select(x => x.Name)))); } foreach (AggregateNode Node in SeparatePromotables) { ECProps.Add(string.Format("PossiblePromotables/{0}={1}", Node.Name, "")); } List <string> ECJobProps = new List <string>(); if (ExplicitTrigger != null) { ECJobProps.Add("IsRoot=0"); } else { ECJobProps.Add("IsRoot=1"); } // here we are just making sure everything before the explicit trigger is completed. if (ExplicitTrigger != null) { foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger)) // if something is already finished, we don't put it into EC { throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name); } } } BuildNode LastSticky = null; bool HitNonSticky = false; bool bHaveECNodes = false; List <string> StepList = new List <string>(); StepList.Add("use strict;"); StepList.Add("use diagnostics;"); StepList.Add("use ElectricCommander();"); StepList.Add("my $ec = new ElectricCommander;"); StepList.Add("$ec->setTimeout(600);"); StepList.Add("my $batch = $ec->newBatch(\"serial\");"); // sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { bHaveECNodes = true; if (NodeToDo.IsSticky) { LastSticky = NodeToDo; if (HitNonSticky && !bSkipTriggers) { throw new AutomationException("Sticky and non-sticky jobs did not sort right."); } } else { HitNonSticky = true; } } } using (CommandUtils.TelemetryStopwatch PerlOutputStopwatch = new CommandUtils.TelemetryStopwatch("PerlOutput")) { string ParentPath = Command.ParseParamValue("ParentPath"); string BaseArgs = String.Format("$batch->createJobStep({{parentPath => '{0}'", ParentPath); bool bHasNoop = false; if (LastSticky == null && bHaveECNodes) { // if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource string Args = String.Format("{0}, subprocedure => 'GUBP_UAT_Node', parallel => '0', jobStepName => 'Noop', actualParameter => [{{actualParameterName => 'NodeName', value => 'Noop'}}, {{actualParameterName => 'Sticky', value =>'1' }}], releaseMode => 'release'}});", BaseArgs); StepList.Add(Args); bHasNoop = true; } Dictionary <string, List <BuildNode> > AgentGroupChains = new Dictionary <string, List <BuildNode> >(); List <BuildNode> StickyChain = new List <BuildNode>(); foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { string MyAgentGroup = NodeToDo.AgentSharingGroup; if (MyAgentGroup != "") { if (!AgentGroupChains.ContainsKey(MyAgentGroup)) { AgentGroupChains.Add(MyAgentGroup, new List <BuildNode> { NodeToDo }); } else { AgentGroupChains[MyAgentGroup].Add(NodeToDo); } } } if (NodeToDo.IsSticky) { if (!StickyChain.Contains(NodeToDo)) { StickyChain.Add(NodeToDo); } } } foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { List <string> NodeProps = GetECPropsForNode(NodeToDo); ECProps.AddRange(NodeProps); bool Sticky = NodeToDo.IsSticky; if (NodeToDo.IsSticky) { if (NodeToDo.AgentSharingGroup != "") { throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name); } if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform); } if (NodeToDo.AgentRequirements != "") { throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name); } } string ProcedureInfix = ""; if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString(); } bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor; TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode; List <Tuple <string, string> > Parameters = new List <Tuple <string, string> >(); Parameters.Add(new Tuple <string, string>("NodeName", NodeToDo.Name)); Parameters.Add(new Tuple <string, string>("Sticky", NodeToDo.IsSticky ? "1" : "0")); if (NodeToDo.AgentSharingGroup != "") { Parameters.Add(new Tuple <string, string>("AgentSharingGroup", NodeToDo.AgentSharingGroup)); } string Procedure; if (TriggerNodeToDo == null || TriggerNodeToDo.IsTriggered) { if (NodeToDo.IsParallelAgentShareEditor) { Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor"; } else { Procedure = "GUBP" + ProcedureInfix + "_UAT_Node"; if (!NodeToDo.IsSticky) { Procedure += "_Parallel"; } if (NodeToDo.AgentSharingGroup != "") { Procedure += "_AgentShare"; } } if (NodeToDo.IsSticky && NodeToDo == LastSticky) { Procedure += "_Release"; } } else { if (TriggerNodeToDo.RequiresRecursiveWorkflow) { Procedure = "GUBP_UAT_Trigger"; //here we run a recursive workflow to wait for the trigger } else { Procedure = "GUBP_Hardcoded_Trigger"; //here we advance the state in the hardcoded workflow so folks can approve } Parameters.Add(new Tuple <string, string>("TriggerState", TriggerNodeToDo.StateName)); Parameters.Add(new Tuple <string, string>("ActionText", TriggerNodeToDo.ActionText)); Parameters.Add(new Tuple <string, string>("DescText", TriggerNodeToDo.DescriptionText)); if (NodeToDo.RecipientsForFailureEmails.Length > 0) { Parameters.Add(new Tuple <string, string>("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails))); } } string ActualParameterArgs = String.Join(", ", Parameters.Select(x => String.Format("{{actualParameterName => '{0}', value => '{1}'}}", x.Item1, x.Item2))); string Args = String.Format("{0}, subprocedure => '{1}', parallel => '{2}', jobStepName => '{3}', actualParameter => [{4}]", BaseArgs, Procedure, DoParallel? 1 : 0, NodeToDo.Name, ActualParameterArgs); List <BuildNode> UncompletedEcDeps = new List <BuildNode>(); { foreach (BuildNode Dep in NodeToDo.AllDirectDependencies) { if (!Dep.IsComplete && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC { if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(NodeToDo)) { throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name); } UncompletedEcDeps.Add(Dep); } } } string PreCondition = GetPreConditionForNode(OrdereredToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps); string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath); string MyAgentGroup = NodeToDo.AgentSharingGroup; bool bDoNestedJobstep = false; bool bDoFirstNestedJobstep = false; string NodeParentPath = ParentPath; if (MyAgentGroup != "") { bDoNestedJobstep = true; NodeParentPath = ParentPath + "/jobSteps[" + MyAgentGroup + "]"; List <BuildNode> MyChain = AgentGroupChains[MyAgentGroup]; int MyIndex = MyChain.IndexOf(NodeToDo); if (MyIndex <= 0) { bDoFirstNestedJobstep = bDoNestedJobstep; } } if (bDoNestedJobstep) { if (bDoFirstNestedJobstep) { { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}', jobStepName => '{1}', parallel => '1'", ParentPath, MyAgentGroup); if (!String.IsNullOrEmpty(PreCondition)) { NestArgs = NestArgs + ", precondition => " + PreCondition; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetPool', subprocedure => 'GUBP{3}_AgentShare_GetPool', parallel => '1', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{4}'}}, {{actualParameterName => 'NodeName', value => '{5}'}}]", ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, MyAgentGroup, NodeToDo.Name); if (!String.IsNullOrEmpty(PreCondition)) { NestArgs = NestArgs + ", precondition => " + PreCondition; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetAgent', subprocedure => 'GUBP{3}_AgentShare_GetAgent', parallel => '1', exclusiveMode => 'call', resourceName => '{4}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{5}'}}, {{actualParameterName => 'NodeName', value=> '{6}'}}]", ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", MyAgentGroup), MyAgentGroup, NodeToDo.Name); { NestArgs = NestArgs + ", precondition => "; NestArgs = NestArgs + "\"\\$\" . \"[/javascript if("; NestArgs = NestArgs + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetPool]/status') == 'completed'"; NestArgs = NestArgs + ") true;]\""; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { PreCondition = "\"\\$\" . \"[/javascript if("; PreCondition = PreCondition + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetAgent]/status') == 'completed'"; PreCondition = PreCondition + ") true;]\""; } } Args = Args.Replace(String.Format("parentPath => '{0}'", ParentPath), String.Format("parentPath => '{0}'", NodeParentPath)); Args = Args.Replace("UAT_Node_Parallel_AgentShare", "UAT_Node_Parallel_AgentShare3"); } if (!String.IsNullOrEmpty(PreCondition)) { Args = Args + ", precondition => " + PreCondition; } if (!String.IsNullOrEmpty(RunCondition)) { Args = Args + ", condition => " + RunCondition; } #if false // this doesn't work because it includes precondition time if (GUBPNodes[NodeToDo].TimeoutInMinutes() > 0) { Args = Args + String.Format(" --timeLimitUnits minutes --timeLimit {0}", GUBPNodes[NodeToDo].TimeoutInMinutes()); } #endif if (Sticky && NodeToDo == LastSticky) { Args = Args + ", releaseMode => 'release'"; } Args = Args + "});"; StepList.Add(Args); if (MyAgentGroup != "" && !bDoNestedJobstep) { List <BuildNode> MyChain = AgentGroupChains[MyAgentGroup]; int MyIndex = MyChain.IndexOf(NodeToDo); if (MyIndex == MyChain.Count - 1) { string RelPreCondition = "\"\\$\" . \"[/javascript if("; // this runs "parallel", but we a precondition to serialize it RelPreCondition = RelPreCondition + "getProperty('" + ParentPath + "/jobSteps[" + NodeToDo.Name + "]/status') == 'completed'"; RelPreCondition = RelPreCondition + ") true;]\""; // we need to release the resource string RelArgs = String.Format("{0}, subprocedure => 'GUBP_Release_AgentShare', parallel => '1', jobStepName => 'Release_{1}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', valued => '{2}'}}], releaseMode => 'release', precondition => '{3}'", BaseArgs, MyAgentGroup, MyAgentGroup, RelPreCondition); StepList.Add(RelArgs); } } } } WriteECPerl(StepList); } bool bHasTests = OrdereredToDo.Any(x => x.Node.IsTest()); RunECTool(String.Format("setProperty \"/myWorkflow/HasTests\" \"{0}\"", bHasTests)); { ECProps.Add("GUBP_LoadedProps=1"); string BranchDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchDef.properties"); CommandUtils.WriteAllLines(BranchDefFile, ECProps.ToArray()); RunECTool(String.Format("setProperty \"/myWorkflow/BranchDefFile\" \"{0}\"", BranchDefFile.Replace("\\", "\\\\"))); } { ECProps.Add("GUBP_LoadedJobProps=1"); string BranchJobDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchJobDef.properties"); CommandUtils.WriteAllLines(BranchJobDefFile, ECProps.ToArray()); RunECTool(String.Format("setProperty \"/myJob/BranchJobDefFile\" \"{0}\"", BranchJobDefFile.Replace("\\", "\\\\"))); } }
/// <summary> /// Main method. /// </summary> /// <param name="CommandLine">Command line</param> public static void Process(string[] CommandLine) { // Initial check for local or build machine runs BEFORE we parse the command line (We need this value set // in case something throws the exception while parsing the command line) IsBuildMachine = !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("uebp_LOCAL_ROOT")); // Scan the command line for commands to execute. var CommandsToExecute = new List <CommandInfo>(); var AdditionalScriptsFolders = new List <string>(); ParseCommandLine(CommandLine, CommandsToExecute, AdditionalScriptsFolders); // Check for build machine override (force local) IsBuildMachine = GlobalCommandLine.ForceLocal ? false : IsBuildMachine; Log.TraceInformation("IsBuildMachine={0}", IsBuildMachine); Environment.SetEnvironmentVariable("IsBuildMachine", IsBuildMachine ? "1" : "0"); // should we kill processes on exit ShouldKillProcesses = !GlobalCommandLine.NoKill; Log.TraceInformation("ShouldKillProcesses={0}", ShouldKillProcesses); if (CommandsToExecute.Count == 0 && GlobalCommandLine.Help) { DisplayHelp(); return; } // Disable AutoSDKs if specified on the command line if (GlobalCommandLine.NoAutoSDK) { UEBuildPlatform.bAllowAutoSDKSwitching = false; } // Setup environment Log.TraceInformation("Setting up command environment."); CommandUtils.InitCommandEnvironment(); // Change CWD to UE4 root. Environment.CurrentDirectory = CommandUtils.CmdEnv.LocalRoot; // Fill in the project info UnrealBuildTool.UProjectInfo.FillProjectInfo(); // Clean rules folders up ProjectUtils.CleanupFolders(); // Compile scripts. Log.TraceInformation("Compiling scripts."); ScriptCompiler Compiler = new ScriptCompiler(); using (CommandUtils.TelemetryStopwatch ScriptCompileStopwatch = new CommandUtils.TelemetryStopwatch("ScriptCompile")) { Compiler.FindAndCompileAllScripts(AdditionalScriptsFolders: AdditionalScriptsFolders); } if (GlobalCommandLine.CompileOnly) { Log.TraceInformation("Compilation successful, exiting (CompileOnly)"); return; } if (GlobalCommandLine.List) { ListAvailableCommands(Compiler.Commands); return; } if (GlobalCommandLine.Help) { DisplayHelp(CommandsToExecute, Compiler.Commands); return; } // Enable or disable P4 support CommandUtils.InitP4Support(CommandsToExecute, Compiler.Commands); if (CommandUtils.P4Enabled) { Log.TraceInformation("Setting up Perforce environment."); CommandUtils.InitP4Environment(); CommandUtils.InitDefaultP4Connection(); } // Find and execute commands. Execute(CommandsToExecute, Compiler.Commands); return; }
public void DoCommanderSetup(IEnumerable<BuildNode> AllNodes, IEnumerable<AggregateNode> AllAggregates, List<BuildNode> OrdereredToDo, List<BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, string CLString, TriggerNode ExplicitTrigger, List<TriggerNode> UnfinishedTriggers, string FakeFail) { List<AggregateNode> SeparatePromotables = FindPromotables(AllAggregates); Dictionary<BuildNode, List<AggregateNode>> DependentPromotions = FindDependentPromotables(AllNodes, SeparatePromotables); Dictionary<BuildNode, int> FullNodeListSortKey = GetDisplayOrder(SortedNodes); if (OrdereredToDo.Count == 0) { throw new AutomationException("No nodes to do!"); } List<string> ECProps = new List<string>(); ECProps.Add(String.Format("TimeIndex={0}", TimeIndex)); foreach (BuildNode Node in SortedNodes.Where(x => x.FrequencyShift != BuildNode.ExplicitFrequencyShift)) { ECProps.Add(string.Format("AllNodes/{0}={1}", Node.Name, GetNodeForAllNodesProperty(Node, TimeQuantum))); } foreach (KeyValuePair<BuildNode, int> NodePair in FullNodeListSortKey.Where(x => x.Key.FrequencyShift != BuildNode.ExplicitFrequencyShift)) { ECProps.Add(string.Format("SortKey/{0}={1}", NodePair.Key.Name, NodePair.Value)); } foreach (KeyValuePair<BuildNode, List<AggregateNode>> NodePair in DependentPromotions) { ECProps.Add(string.Format("DependentPromotions/{0}={1}", NodePair.Key.Name, String.Join(" ", NodePair.Value.Select(x => x.Name)))); } foreach (AggregateNode Node in SeparatePromotables) { ECProps.Add(string.Format("PossiblePromotables/{0}={1}", Node.Name, "")); } List<string> ECJobProps = new List<string>(); if (ExplicitTrigger != null) { ECJobProps.Add("IsRoot=0"); } else { ECJobProps.Add("IsRoot=1"); } // here we are just making sure everything before the explicit trigger is completed. if (ExplicitTrigger != null) { foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger)) // if something is already finished, we don't put it into EC { throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name); } } } BuildNode LastSticky = null; bool HitNonSticky = false; bool bHaveECNodes = false; List<string> StepList = new List<string>(); StepList.Add("use strict;"); StepList.Add("use diagnostics;"); StepList.Add("use ElectricCommander();"); StepList.Add("my $ec = new ElectricCommander;"); StepList.Add("$ec->setTimeout(600);"); StepList.Add("my $batch = $ec->newBatch(\"serial\");"); // sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { bHaveECNodes = true; if (NodeToDo.IsSticky) { LastSticky = NodeToDo; if (HitNonSticky && !bSkipTriggers) { throw new AutomationException("Sticky and non-sticky jobs did not sort right."); } } else { HitNonSticky = true; } } } using(CommandUtils.TelemetryStopwatch PerlOutputStopwatch = new CommandUtils.TelemetryStopwatch("PerlOutput")) { string ParentPath = Command.ParseParamValue("ParentPath"); string BaseArgs = String.Format("$batch->createJobStep({{parentPath => '{0}'", ParentPath); bool bHasNoop = false; if (LastSticky == null && bHaveECNodes) { // if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource string Args = String.Format("{0}, subprocedure => 'GUBP_UAT_Node', parallel => '0', jobStepName => 'Noop', actualParameter => [{{actualParameterName => 'NodeName', value => 'Noop'}}, {{actualParameterName => 'Sticky', value =>'1' }}], releaseMode => 'release'}});", BaseArgs); StepList.Add(Args); bHasNoop = true; } Dictionary<string, List<BuildNode>> AgentGroupChains = new Dictionary<string, List<BuildNode>>(); List<BuildNode> StickyChain = new List<BuildNode>(); foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { string MyAgentGroup = NodeToDo.AgentSharingGroup; if (MyAgentGroup != "") { if (!AgentGroupChains.ContainsKey(MyAgentGroup)) { AgentGroupChains.Add(MyAgentGroup, new List<BuildNode> { NodeToDo }); } else { AgentGroupChains[MyAgentGroup].Add(NodeToDo); } } } if (NodeToDo.IsSticky) { if (!StickyChain.Contains(NodeToDo)) { StickyChain.Add(NodeToDo); } } } foreach (BuildNode NodeToDo in OrdereredToDo) { if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC { List<string> NodeProps = GetECPropsForNode(NodeToDo); ECProps.AddRange(NodeProps); bool Sticky = NodeToDo.IsSticky; if(NodeToDo.IsSticky) { if(NodeToDo.AgentSharingGroup != "") { throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name); } if(NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform); } if(NodeToDo.AgentRequirements != "") { throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name); } } string ProcedureInfix = ""; if(NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64) { ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString(); } bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor; TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode; List<Tuple<string, string>> Parameters = new List<Tuple<string,string>>(); Parameters.Add(new Tuple<string, string>("NodeName", NodeToDo.Name)); Parameters.Add(new Tuple<string, string>("Sticky", NodeToDo.IsSticky ? "1" : "0")); if (NodeToDo.AgentSharingGroup != "") { Parameters.Add(new Tuple<string, string>("AgentSharingGroup", NodeToDo.AgentSharingGroup)); } string Procedure; if(TriggerNodeToDo == null || TriggerNodeToDo.IsTriggered) { if (NodeToDo.IsParallelAgentShareEditor) { Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor"; } else { Procedure = "GUBP" + ProcedureInfix + "_UAT_Node"; if (!NodeToDo.IsSticky) { Procedure += "_Parallel"; } if (NodeToDo.AgentSharingGroup != "") { Procedure += "_AgentShare"; } } if (NodeToDo.IsSticky && NodeToDo == LastSticky) { Procedure += "_Release"; } } else { if(TriggerNodeToDo.RequiresRecursiveWorkflow) { Procedure = "GUBP_UAT_Trigger"; //here we run a recursive workflow to wait for the trigger } else { Procedure = "GUBP_Hardcoded_Trigger"; //here we advance the state in the hardcoded workflow so folks can approve } Parameters.Add(new Tuple<string, string>("TriggerState", TriggerNodeToDo.StateName)); Parameters.Add(new Tuple<string, string>("ActionText", TriggerNodeToDo.ActionText)); Parameters.Add(new Tuple<string, string>("DescText", TriggerNodeToDo.DescriptionText)); if (NodeToDo.RecipientsForFailureEmails.Length > 0) { Parameters.Add(new Tuple<string, string>("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails))); } } string ActualParameterArgs = String.Join(", ", Parameters.Select(x => String.Format("{{actualParameterName => '{0}', value => '{1}'}}", x.Item1, x.Item2))); string Args = String.Format("{0}, subprocedure => '{1}', parallel => '{2}', jobStepName => '{3}', actualParameter => [{4}]", BaseArgs, Procedure, DoParallel? 1 : 0, NodeToDo.Name, ActualParameterArgs); List<BuildNode> UncompletedEcDeps = new List<BuildNode>(); { foreach (BuildNode Dep in NodeToDo.AllDirectDependencies) { if (!Dep.IsComplete && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC { if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(NodeToDo)) { throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name); } UncompletedEcDeps.Add(Dep); } } } string PreCondition = GetPreConditionForNode(OrdereredToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps); string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath); string MyAgentGroup = NodeToDo.AgentSharingGroup; bool bDoNestedJobstep = false; bool bDoFirstNestedJobstep = false; string NodeParentPath = ParentPath; if (MyAgentGroup != "") { bDoNestedJobstep = true; NodeParentPath = ParentPath + "/jobSteps[" + MyAgentGroup + "]"; List<BuildNode> MyChain = AgentGroupChains[MyAgentGroup]; int MyIndex = MyChain.IndexOf(NodeToDo); if (MyIndex <= 0) { bDoFirstNestedJobstep = bDoNestedJobstep; } } if (bDoNestedJobstep) { if (bDoFirstNestedJobstep) { { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}', jobStepName => '{1}', parallel => '1'", ParentPath, MyAgentGroup); if (!String.IsNullOrEmpty(PreCondition)) { NestArgs = NestArgs + ", precondition => " + PreCondition; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetPool', subprocedure => 'GUBP{3}_AgentShare_GetPool', parallel => '1', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{4}'}}, {{actualParameterName => 'NodeName', value => '{5}'}}]", ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, MyAgentGroup, NodeToDo.Name); if (!String.IsNullOrEmpty(PreCondition)) { NestArgs = NestArgs + ", precondition => " + PreCondition; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetAgent', subprocedure => 'GUBP{3}_AgentShare_GetAgent', parallel => '1', exclusiveMode => 'call', resourceName => '{4}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{5}'}}, {{actualParameterName => 'NodeName', value=> '{6}'}}]", ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", MyAgentGroup), MyAgentGroup, NodeToDo.Name); { NestArgs = NestArgs + ", precondition => "; NestArgs = NestArgs + "\"\\$\" . \"[/javascript if("; NestArgs = NestArgs + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetPool]/status') == 'completed'"; NestArgs = NestArgs + ") true;]\""; } NestArgs = NestArgs + "});"; StepList.Add(NestArgs); } { PreCondition = "\"\\$\" . \"[/javascript if("; PreCondition = PreCondition + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetAgent]/status') == 'completed'"; PreCondition = PreCondition + ") true;]\""; } } Args = Args.Replace(String.Format("parentPath => '{0}'", ParentPath), String.Format("parentPath => '{0}'", NodeParentPath)); Args = Args.Replace("UAT_Node_Parallel_AgentShare", "UAT_Node_Parallel_AgentShare3"); } if (!String.IsNullOrEmpty(PreCondition)) { Args = Args + ", precondition => " + PreCondition; } if (!String.IsNullOrEmpty(RunCondition)) { Args = Args + ", condition => " + RunCondition; } #if false // this doesn't work because it includes precondition time if (GUBPNodes[NodeToDo].TimeoutInMinutes() > 0) { Args = Args + String.Format(" --timeLimitUnits minutes --timeLimit {0}", GUBPNodes[NodeToDo].TimeoutInMinutes()); } #endif if (Sticky && NodeToDo == LastSticky) { Args = Args + ", releaseMode => 'release'"; } Args = Args + "});"; StepList.Add(Args); if (MyAgentGroup != "" && !bDoNestedJobstep) { List<BuildNode> MyChain = AgentGroupChains[MyAgentGroup]; int MyIndex = MyChain.IndexOf(NodeToDo); if (MyIndex == MyChain.Count - 1) { string RelPreCondition = "\"\\$\" . \"[/javascript if("; // this runs "parallel", but we a precondition to serialize it RelPreCondition = RelPreCondition + "getProperty('" + ParentPath + "/jobSteps[" + NodeToDo.Name + "]/status') == 'completed'"; RelPreCondition = RelPreCondition + ") true;]\""; // we need to release the resource string RelArgs = String.Format("{0}, subprocedure => 'GUBP_Release_AgentShare', parallel => '1', jobStepName => 'Release_{1}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', valued => '{2}'}}], releaseMode => 'release', precondition => '{3}'", BaseArgs, MyAgentGroup, MyAgentGroup, RelPreCondition); StepList.Add(RelArgs); } } } } WriteECPerl(StepList); } bool bHasTests = OrdereredToDo.Any(x => x.Node.IsTest()); RunECTool(String.Format("setProperty \"/myWorkflow/HasTests\" \"{0}\"", bHasTests)); { ECProps.Add("GUBP_LoadedProps=1"); string BranchDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchDef.properties"); CommandUtils.WriteAllLines(BranchDefFile, ECProps.ToArray()); RunECTool(String.Format("setProperty \"/myWorkflow/BranchDefFile\" \"{0}\"", BranchDefFile.Replace("\\", "\\\\"))); } { ECProps.Add("GUBP_LoadedJobProps=1"); string BranchJobDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchJobDef.properties"); CommandUtils.WriteAllLines(BranchJobDefFile, ECProps.ToArray()); RunECTool(String.Format("setProperty \"/myJob/BranchJobDefFile\" \"{0}\"", BranchJobDefFile.Replace("\\", "\\\\"))); } }
/// <summary> /// Main method. /// </summary> /// <param name="CommandLine">Command line</param> public static void Process(string[] CommandLine) { // Initial check for local or build machine runs BEFORE we parse the command line (We need this value set // in case something throws the exception while parsing the command line) IsBuildMachine = !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("uebp_LOCAL_ROOT")); // Scan the command line for commands to execute. var CommandsToExecute = new List<CommandInfo>(); var AdditionalScriptsFolders = new List<string>(); ParseCommandLine(CommandLine, CommandsToExecute, AdditionalScriptsFolders); // Check for build machine override (force local) IsBuildMachine = GlobalCommandLine.ForceLocal ? false : IsBuildMachine; Log.TraceInformation("IsBuildMachine={0}", IsBuildMachine); Environment.SetEnvironmentVariable("IsBuildMachine", IsBuildMachine ? "1" : "0"); // should we kill processes on exit ShouldKillProcesses = !GlobalCommandLine.NoKill; Log.TraceInformation("ShouldKillProcesses={0}", ShouldKillProcesses); if (CommandsToExecute.Count == 0 && GlobalCommandLine.Help) { DisplayHelp(); return; } // Disable AutoSDKs if specified on the command line if (GlobalCommandLine.NoAutoSDK) { UEBuildPlatform.bAllowAutoSDKSwitching = false; } // Setup environment Log.TraceInformation("Setting up command environment."); CommandUtils.InitCommandEnvironment(); // Change CWD to UE4 root. Environment.CurrentDirectory = CommandUtils.CmdEnv.LocalRoot; // Fill in the project info UnrealBuildTool.UProjectInfo.FillProjectInfo(); // Clean rules folders up ProjectUtils.CleanupFolders(); // Compile scripts. Log.TraceInformation("Compiling scripts."); ScriptCompiler Compiler = new ScriptCompiler(); using(CommandUtils.TelemetryStopwatch ScriptCompileStopwatch = new CommandUtils.TelemetryStopwatch("ScriptCompile")) { Compiler.FindAndCompileAllScripts(AdditionalScriptsFolders: AdditionalScriptsFolders); } if (GlobalCommandLine.CompileOnly) { Log.TraceInformation("Compilation successful, exiting (CompileOnly)"); return; } if (GlobalCommandLine.List) { ListAvailableCommands(Compiler.Commands); return; } if (GlobalCommandLine.Help) { DisplayHelp(CommandsToExecute, Compiler.Commands); return; } // Enable or disable P4 support CommandUtils.InitP4Support(CommandsToExecute, Compiler.Commands); if (CommandUtils.P4Enabled) { Log.TraceInformation("Setting up Perforce environment."); CommandUtils.InitP4Environment(); CommandUtils.InitDefaultP4Connection(); } // Find and execute commands. Execute(CommandsToExecute, Compiler.Commands); return; }