public BranchInfo(List <UnrealTargetPlatform> InHostPlatforms)
{
BaseEngineProject = new BranchUProject();
var AllProjects = UnrealBuildTool.UProjectInfo.FilterGameProjects(false, null);
using (CommandUtils.TelemetryStopwatch SortProjectsStopwatch = new CommandUtils.TelemetryStopwatch("SortProjects"))
{
foreach (var InfoEntry in AllProjects)
{
var UProject = new BranchUProject(InfoEntry);
if (UProject.Properties.bIsCodeBasedProject)
{
CodeProjects.Add(UProject);
}
else
{
NonCodeProjects.Add(UProject);
// the base project uses BlankProject if it really needs a .uproject file
if (String.IsNullOrEmpty(BaseEngineProject.FilePath) && UProject.GameName == "BlankProject")
{
BaseEngineProject.FilePath = UProject.FilePath;
}
}
}
}
/* if (String.IsNullOrEmpty(BaseEngineProject.FilePath))
* {
* throw new AutomationException("All branches must have the blank project /Samples/Sandbox/BlankProject");
* }*/
using (CommandUtils.TelemetryStopwatch ProjectDumpStopwatch = new CommandUtils.TelemetryStopwatch("Project Dump"))
{
CommandUtils.Log(" Base Engine:");
BaseEngineProject.Dump(InHostPlatforms);
CommandUtils.Log(" {0} Code projects:", CodeProjects.Count);
foreach (var Proj in CodeProjects)
{
Proj.Dump(InHostPlatforms);
}
CommandUtils.Log(" {0} Non-Code projects:", CodeProjects.Count);
foreach (var Proj in NonCodeProjects)
{
Proj.Dump(InHostPlatforms);
}
}
}
public BranchInfo(List<UnrealTargetPlatform> InHostPlatforms)
{
BaseEngineProject = new BranchUProject();
var AllProjects = UnrealBuildTool.UProjectInfo.FilterGameProjects(false, null);
using(CommandUtils.TelemetryStopwatch SortProjectsStopwatch = new CommandUtils.TelemetryStopwatch("SortProjects"))
{
foreach (var InfoEntry in AllProjects)
{
var UProject = new BranchUProject(InfoEntry);
if (UProject.Properties.bIsCodeBasedProject)
{
CodeProjects.Add(UProject);
}
else
{
NonCodeProjects.Add(UProject);
// the base project uses BlankProject if it really needs a .uproject file
if (String.IsNullOrEmpty(BaseEngineProject.FilePath) && UProject.GameName == "BlankProject")
{
BaseEngineProject.FilePath = UProject.FilePath;
}
}
}
}
/* if (String.IsNullOrEmpty(BaseEngineProject.FilePath))
{
throw new AutomationException("All branches must have the blank project /Samples/Sandbox/BlankProject");
}*/
using(CommandUtils.TelemetryStopwatch ProjectDumpStopwatch = new CommandUtils.TelemetryStopwatch("Project Dump"))
{
CommandUtils.Log(" Base Engine:");
BaseEngineProject.Dump(InHostPlatforms);
CommandUtils.Log(" {0} Code projects:", CodeProjects.Count);
foreach (var Proj in CodeProjects)
{
Proj.Dump(InHostPlatforms);
}
CommandUtils.Log(" {0} Non-Code projects:", CodeProjects.Count);
foreach (var Proj in NonCodeProjects)
{
Proj.Dump(InHostPlatforms);
}
}
}
public void DoCommanderSetup(IEnumerable <BuildNode> AllNodes, IEnumerable <AggregateNode> AllAggregates, List <BuildNode> OrdereredToDo, List <BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, string CLString, TriggerNode ExplicitTrigger, List <TriggerNode> UnfinishedTriggers, string FakeFail)
{
List <AggregateNode> SeparatePromotables = FindPromotables(AllAggregates);
Dictionary <BuildNode, List <AggregateNode> > DependentPromotions = FindDependentPromotables(AllNodes, SeparatePromotables);
Dictionary <BuildNode, int> FullNodeListSortKey = GetDisplayOrder(SortedNodes);
if (OrdereredToDo.Count == 0)
{
throw new AutomationException("No nodes to do!");
}
List <string> ECProps = new List <string>();
ECProps.Add(String.Format("TimeIndex={0}", TimeIndex));
foreach (BuildNode Node in SortedNodes.Where(x => x.FrequencyShift != BuildNode.ExplicitFrequencyShift))
{
ECProps.Add(string.Format("AllNodes/{0}={1}", Node.Name, GetNodeForAllNodesProperty(Node, TimeQuantum)));
}
foreach (KeyValuePair <BuildNode, int> NodePair in FullNodeListSortKey.Where(x => x.Key.FrequencyShift != BuildNode.ExplicitFrequencyShift))
{
ECProps.Add(string.Format("SortKey/{0}={1}", NodePair.Key.Name, NodePair.Value));
}
foreach (KeyValuePair <BuildNode, List <AggregateNode> > NodePair in DependentPromotions)
{
ECProps.Add(string.Format("DependentPromotions/{0}={1}", NodePair.Key.Name, String.Join(" ", NodePair.Value.Select(x => x.Name))));
}
foreach (AggregateNode Node in SeparatePromotables)
{
ECProps.Add(string.Format("PossiblePromotables/{0}={1}", Node.Name, ""));
}
List <string> ECJobProps = new List <string>();
if (ExplicitTrigger != null)
{
ECJobProps.Add("IsRoot=0");
}
else
{
ECJobProps.Add("IsRoot=1");
}
// here we are just making sure everything before the explicit trigger is completed.
if (ExplicitTrigger != null)
{
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger)) // if something is already finished, we don't put it into EC
{
throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name);
}
}
}
BuildNode LastSticky = null;
bool HitNonSticky = false;
bool bHaveECNodes = false;
List <string> StepList = new List <string>();
StepList.Add("use strict;");
StepList.Add("use diagnostics;");
StepList.Add("use ElectricCommander();");
StepList.Add("my $ec = new ElectricCommander;");
StepList.Add("$ec->setTimeout(600);");
StepList.Add("my $batch = $ec->newBatch(\"serial\");");
// sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
bHaveECNodes = true;
if (NodeToDo.IsSticky)
{
LastSticky = NodeToDo;
if (HitNonSticky && !bSkipTriggers)
{
throw new AutomationException("Sticky and non-sticky jobs did not sort right.");
}
}
else
{
HitNonSticky = true;
}
}
}
using (CommandUtils.TelemetryStopwatch PerlOutputStopwatch = new CommandUtils.TelemetryStopwatch("PerlOutput"))
{
string ParentPath = Command.ParseParamValue("ParentPath");
string BaseArgs = String.Format("$batch->createJobStep({{parentPath => '{0}'", ParentPath);
bool bHasNoop = false;
if (LastSticky == null && bHaveECNodes)
{
// if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource
string Args = String.Format("{0}, subprocedure => 'GUBP_UAT_Node', parallel => '0', jobStepName => 'Noop', actualParameter => [{{actualParameterName => 'NodeName', value => 'Noop'}}, {{actualParameterName => 'Sticky', value =>'1' }}], releaseMode => 'release'}});", BaseArgs);
StepList.Add(Args);
bHasNoop = true;
}
Dictionary <string, List <BuildNode> > AgentGroupChains = new Dictionary <string, List <BuildNode> >();
List <BuildNode> StickyChain = new List <BuildNode>();
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
string MyAgentGroup = NodeToDo.AgentSharingGroup;
if (MyAgentGroup != "")
{
if (!AgentGroupChains.ContainsKey(MyAgentGroup))
{
AgentGroupChains.Add(MyAgentGroup, new List <BuildNode> {
NodeToDo
});
}
else
{
AgentGroupChains[MyAgentGroup].Add(NodeToDo);
}
}
}
if (NodeToDo.IsSticky)
{
if (!StickyChain.Contains(NodeToDo))
{
StickyChain.Add(NodeToDo);
}
}
}
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
List <string> NodeProps = GetECPropsForNode(NodeToDo);
ECProps.AddRange(NodeProps);
bool Sticky = NodeToDo.IsSticky;
if (NodeToDo.IsSticky)
{
if (NodeToDo.AgentSharingGroup != "")
{
throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name);
}
if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
{
throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform);
}
if (NodeToDo.AgentRequirements != "")
{
throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name);
}
}
string ProcedureInfix = "";
if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
{
ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString();
}
bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor;
TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode;
List <Tuple <string, string> > Parameters = new List <Tuple <string, string> >();
Parameters.Add(new Tuple <string, string>("NodeName", NodeToDo.Name));
Parameters.Add(new Tuple <string, string>("Sticky", NodeToDo.IsSticky ? "1" : "0"));
if (NodeToDo.AgentSharingGroup != "")
{
Parameters.Add(new Tuple <string, string>("AgentSharingGroup", NodeToDo.AgentSharingGroup));
}
string Procedure;
if (TriggerNodeToDo == null || TriggerNodeToDo.IsTriggered)
{
if (NodeToDo.IsParallelAgentShareEditor)
{
Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor";
}
else
{
Procedure = "GUBP" + ProcedureInfix + "_UAT_Node";
if (!NodeToDo.IsSticky)
{
Procedure += "_Parallel";
}
if (NodeToDo.AgentSharingGroup != "")
{
Procedure += "_AgentShare";
}
}
if (NodeToDo.IsSticky && NodeToDo == LastSticky)
{
Procedure += "_Release";
}
}
else
{
if (TriggerNodeToDo.RequiresRecursiveWorkflow)
{
Procedure = "GUBP_UAT_Trigger"; //here we run a recursive workflow to wait for the trigger
}
else
{
Procedure = "GUBP_Hardcoded_Trigger"; //here we advance the state in the hardcoded workflow so folks can approve
}
Parameters.Add(new Tuple <string, string>("TriggerState", TriggerNodeToDo.StateName));
Parameters.Add(new Tuple <string, string>("ActionText", TriggerNodeToDo.ActionText));
Parameters.Add(new Tuple <string, string>("DescText", TriggerNodeToDo.DescriptionText));
if (NodeToDo.RecipientsForFailureEmails.Length > 0)
{
Parameters.Add(new Tuple <string, string>("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails)));
}
}
string ActualParameterArgs = String.Join(", ", Parameters.Select(x => String.Format("{{actualParameterName => '{0}', value => '{1}'}}", x.Item1, x.Item2)));
string Args = String.Format("{0}, subprocedure => '{1}', parallel => '{2}', jobStepName => '{3}', actualParameter => [{4}]", BaseArgs, Procedure, DoParallel? 1 : 0, NodeToDo.Name, ActualParameterArgs);
List <BuildNode> UncompletedEcDeps = new List <BuildNode>();
{
foreach (BuildNode Dep in NodeToDo.AllDirectDependencies)
{
if (!Dep.IsComplete && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC
{
if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(NodeToDo))
{
throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name);
}
UncompletedEcDeps.Add(Dep);
}
}
}
string PreCondition = GetPreConditionForNode(OrdereredToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps);
string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath);
string MyAgentGroup = NodeToDo.AgentSharingGroup;
bool bDoNestedJobstep = false;
bool bDoFirstNestedJobstep = false;
string NodeParentPath = ParentPath;
if (MyAgentGroup != "")
{
bDoNestedJobstep = true;
NodeParentPath = ParentPath + "/jobSteps[" + MyAgentGroup + "]";
List <BuildNode> MyChain = AgentGroupChains[MyAgentGroup];
int MyIndex = MyChain.IndexOf(NodeToDo);
if (MyIndex <= 0)
{
bDoFirstNestedJobstep = bDoNestedJobstep;
}
}
if (bDoNestedJobstep)
{
if (bDoFirstNestedJobstep)
{
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}', jobStepName => '{1}', parallel => '1'",
ParentPath, MyAgentGroup);
if (!String.IsNullOrEmpty(PreCondition))
{
NestArgs = NestArgs + ", precondition => " + PreCondition;
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetPool', subprocedure => 'GUBP{3}_AgentShare_GetPool', parallel => '1', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{4}'}}, {{actualParameterName => 'NodeName', value => '{5}'}}]",
ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, MyAgentGroup, NodeToDo.Name);
if (!String.IsNullOrEmpty(PreCondition))
{
NestArgs = NestArgs + ", precondition => " + PreCondition;
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetAgent', subprocedure => 'GUBP{3}_AgentShare_GetAgent', parallel => '1', exclusiveMode => 'call', resourceName => '{4}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{5}'}}, {{actualParameterName => 'NodeName', value=> '{6}'}}]",
ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix,
String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", MyAgentGroup),
MyAgentGroup, NodeToDo.Name);
{
NestArgs = NestArgs + ", precondition => ";
NestArgs = NestArgs + "\"\\$\" . \"[/javascript if(";
NestArgs = NestArgs + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetPool]/status') == 'completed'";
NestArgs = NestArgs + ") true;]\"";
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
PreCondition = "\"\\$\" . \"[/javascript if(";
PreCondition = PreCondition + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetAgent]/status') == 'completed'";
PreCondition = PreCondition + ") true;]\"";
}
}
Args = Args.Replace(String.Format("parentPath => '{0}'", ParentPath), String.Format("parentPath => '{0}'", NodeParentPath));
Args = Args.Replace("UAT_Node_Parallel_AgentShare", "UAT_Node_Parallel_AgentShare3");
}
if (!String.IsNullOrEmpty(PreCondition))
{
Args = Args + ", precondition => " + PreCondition;
}
if (!String.IsNullOrEmpty(RunCondition))
{
Args = Args + ", condition => " + RunCondition;
}
#if false
// this doesn't work because it includes precondition time
if (GUBPNodes[NodeToDo].TimeoutInMinutes() > 0)
{
Args = Args + String.Format(" --timeLimitUnits minutes --timeLimit {0}", GUBPNodes[NodeToDo].TimeoutInMinutes());
}
#endif
if (Sticky && NodeToDo == LastSticky)
{
Args = Args + ", releaseMode => 'release'";
}
Args = Args + "});";
StepList.Add(Args);
if (MyAgentGroup != "" && !bDoNestedJobstep)
{
List <BuildNode> MyChain = AgentGroupChains[MyAgentGroup];
int MyIndex = MyChain.IndexOf(NodeToDo);
if (MyIndex == MyChain.Count - 1)
{
string RelPreCondition = "\"\\$\" . \"[/javascript if(";
// this runs "parallel", but we a precondition to serialize it
RelPreCondition = RelPreCondition + "getProperty('" + ParentPath + "/jobSteps[" + NodeToDo.Name + "]/status') == 'completed'";
RelPreCondition = RelPreCondition + ") true;]\"";
// we need to release the resource
string RelArgs = String.Format("{0}, subprocedure => 'GUBP_Release_AgentShare', parallel => '1', jobStepName => 'Release_{1}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', valued => '{2}'}}], releaseMode => 'release', precondition => '{3}'",
BaseArgs, MyAgentGroup, MyAgentGroup, RelPreCondition);
StepList.Add(RelArgs);
}
}
}
}
WriteECPerl(StepList);
}
bool bHasTests = OrdereredToDo.Any(x => x.Node.IsTest());
RunECTool(String.Format("setProperty \"/myWorkflow/HasTests\" \"{0}\"", bHasTests));
{
ECProps.Add("GUBP_LoadedProps=1");
string BranchDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchDef.properties");
CommandUtils.WriteAllLines(BranchDefFile, ECProps.ToArray());
RunECTool(String.Format("setProperty \"/myWorkflow/BranchDefFile\" \"{0}\"", BranchDefFile.Replace("\\", "\\\\")));
}
{
ECProps.Add("GUBP_LoadedJobProps=1");
string BranchJobDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchJobDef.properties");
CommandUtils.WriteAllLines(BranchJobDefFile, ECProps.ToArray());
RunECTool(String.Format("setProperty \"/myJob/BranchJobDefFile\" \"{0}\"", BranchJobDefFile.Replace("\\", "\\\\")));
}
}
/// <summary>
/// Main method.
/// </summary>
/// <param name="CommandLine">Command line</param>
public static void Process(string[] CommandLine)
{
// Initial check for local or build machine runs BEFORE we parse the command line (We need this value set
// in case something throws the exception while parsing the command line)
IsBuildMachine = !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("uebp_LOCAL_ROOT"));
// Scan the command line for commands to execute.
var CommandsToExecute = new List <CommandInfo>();
var AdditionalScriptsFolders = new List <string>();
ParseCommandLine(CommandLine, CommandsToExecute, AdditionalScriptsFolders);
// Check for build machine override (force local)
IsBuildMachine = GlobalCommandLine.ForceLocal ? false : IsBuildMachine;
Log.TraceInformation("IsBuildMachine={0}", IsBuildMachine);
Environment.SetEnvironmentVariable("IsBuildMachine", IsBuildMachine ? "1" : "0");
// should we kill processes on exit
ShouldKillProcesses = !GlobalCommandLine.NoKill;
Log.TraceInformation("ShouldKillProcesses={0}", ShouldKillProcesses);
if (CommandsToExecute.Count == 0 && GlobalCommandLine.Help)
{
DisplayHelp();
return;
}
// Disable AutoSDKs if specified on the command line
if (GlobalCommandLine.NoAutoSDK)
{
UEBuildPlatform.bAllowAutoSDKSwitching = false;
}
// Setup environment
Log.TraceInformation("Setting up command environment.");
CommandUtils.InitCommandEnvironment();
// Change CWD to UE4 root.
Environment.CurrentDirectory = CommandUtils.CmdEnv.LocalRoot;
// Fill in the project info
UnrealBuildTool.UProjectInfo.FillProjectInfo();
// Clean rules folders up
ProjectUtils.CleanupFolders();
// Compile scripts.
Log.TraceInformation("Compiling scripts.");
ScriptCompiler Compiler = new ScriptCompiler();
using (CommandUtils.TelemetryStopwatch ScriptCompileStopwatch = new CommandUtils.TelemetryStopwatch("ScriptCompile"))
{
Compiler.FindAndCompileAllScripts(AdditionalScriptsFolders: AdditionalScriptsFolders);
}
if (GlobalCommandLine.CompileOnly)
{
Log.TraceInformation("Compilation successful, exiting (CompileOnly)");
return;
}
if (GlobalCommandLine.List)
{
ListAvailableCommands(Compiler.Commands);
return;
}
if (GlobalCommandLine.Help)
{
DisplayHelp(CommandsToExecute, Compiler.Commands);
return;
}
// Enable or disable P4 support
CommandUtils.InitP4Support(CommandsToExecute, Compiler.Commands);
if (CommandUtils.P4Enabled)
{
Log.TraceInformation("Setting up Perforce environment.");
CommandUtils.InitP4Environment();
CommandUtils.InitDefaultP4Connection();
}
// Find and execute commands.
Execute(CommandsToExecute, Compiler.Commands);
return;
}
public void DoCommanderSetup(IEnumerable<BuildNode> AllNodes, IEnumerable<AggregateNode> AllAggregates, List<BuildNode> OrdereredToDo, List<BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, string CLString, TriggerNode ExplicitTrigger, List<TriggerNode> UnfinishedTriggers, string FakeFail)
{
List<AggregateNode> SeparatePromotables = FindPromotables(AllAggregates);
Dictionary<BuildNode, List<AggregateNode>> DependentPromotions = FindDependentPromotables(AllNodes, SeparatePromotables);
Dictionary<BuildNode, int> FullNodeListSortKey = GetDisplayOrder(SortedNodes);
if (OrdereredToDo.Count == 0)
{
throw new AutomationException("No nodes to do!");
}
List<string> ECProps = new List<string>();
ECProps.Add(String.Format("TimeIndex={0}", TimeIndex));
foreach (BuildNode Node in SortedNodes.Where(x => x.FrequencyShift != BuildNode.ExplicitFrequencyShift))
{
ECProps.Add(string.Format("AllNodes/{0}={1}", Node.Name, GetNodeForAllNodesProperty(Node, TimeQuantum)));
}
foreach (KeyValuePair<BuildNode, int> NodePair in FullNodeListSortKey.Where(x => x.Key.FrequencyShift != BuildNode.ExplicitFrequencyShift))
{
ECProps.Add(string.Format("SortKey/{0}={1}", NodePair.Key.Name, NodePair.Value));
}
foreach (KeyValuePair<BuildNode, List<AggregateNode>> NodePair in DependentPromotions)
{
ECProps.Add(string.Format("DependentPromotions/{0}={1}", NodePair.Key.Name, String.Join(" ", NodePair.Value.Select(x => x.Name))));
}
foreach (AggregateNode Node in SeparatePromotables)
{
ECProps.Add(string.Format("PossiblePromotables/{0}={1}", Node.Name, ""));
}
List<string> ECJobProps = new List<string>();
if (ExplicitTrigger != null)
{
ECJobProps.Add("IsRoot=0");
}
else
{
ECJobProps.Add("IsRoot=1");
}
// here we are just making sure everything before the explicit trigger is completed.
if (ExplicitTrigger != null)
{
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger)) // if something is already finished, we don't put it into EC
{
throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name);
}
}
}
BuildNode LastSticky = null;
bool HitNonSticky = false;
bool bHaveECNodes = false;
List<string> StepList = new List<string>();
StepList.Add("use strict;");
StepList.Add("use diagnostics;");
StepList.Add("use ElectricCommander();");
StepList.Add("my $ec = new ElectricCommander;");
StepList.Add("$ec->setTimeout(600);");
StepList.Add("my $batch = $ec->newBatch(\"serial\");");
// sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
bHaveECNodes = true;
if (NodeToDo.IsSticky)
{
LastSticky = NodeToDo;
if (HitNonSticky && !bSkipTriggers)
{
throw new AutomationException("Sticky and non-sticky jobs did not sort right.");
}
}
else
{
HitNonSticky = true;
}
}
}
using(CommandUtils.TelemetryStopwatch PerlOutputStopwatch = new CommandUtils.TelemetryStopwatch("PerlOutput"))
{
string ParentPath = Command.ParseParamValue("ParentPath");
string BaseArgs = String.Format("$batch->createJobStep({{parentPath => '{0}'", ParentPath);
bool bHasNoop = false;
if (LastSticky == null && bHaveECNodes)
{
// if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource
string Args = String.Format("{0}, subprocedure => 'GUBP_UAT_Node', parallel => '0', jobStepName => 'Noop', actualParameter => [{{actualParameterName => 'NodeName', value => 'Noop'}}, {{actualParameterName => 'Sticky', value =>'1' }}], releaseMode => 'release'}});", BaseArgs);
StepList.Add(Args);
bHasNoop = true;
}
Dictionary<string, List<BuildNode>> AgentGroupChains = new Dictionary<string, List<BuildNode>>();
List<BuildNode> StickyChain = new List<BuildNode>();
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
string MyAgentGroup = NodeToDo.AgentSharingGroup;
if (MyAgentGroup != "")
{
if (!AgentGroupChains.ContainsKey(MyAgentGroup))
{
AgentGroupChains.Add(MyAgentGroup, new List<BuildNode> { NodeToDo });
}
else
{
AgentGroupChains[MyAgentGroup].Add(NodeToDo);
}
}
}
if (NodeToDo.IsSticky)
{
if (!StickyChain.Contains(NodeToDo))
{
StickyChain.Add(NodeToDo);
}
}
}
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
List<string> NodeProps = GetECPropsForNode(NodeToDo);
ECProps.AddRange(NodeProps);
bool Sticky = NodeToDo.IsSticky;
if(NodeToDo.IsSticky)
{
if(NodeToDo.AgentSharingGroup != "")
{
throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name);
}
if(NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
{
throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform);
}
if(NodeToDo.AgentRequirements != "")
{
throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name);
}
}
string ProcedureInfix = "";
if(NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
{
ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString();
}
bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor;
TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode;
List<Tuple<string, string>> Parameters = new List<Tuple<string,string>>();
Parameters.Add(new Tuple<string, string>("NodeName", NodeToDo.Name));
Parameters.Add(new Tuple<string, string>("Sticky", NodeToDo.IsSticky ? "1" : "0"));
if (NodeToDo.AgentSharingGroup != "")
{
Parameters.Add(new Tuple<string, string>("AgentSharingGroup", NodeToDo.AgentSharingGroup));
}
string Procedure;
if(TriggerNodeToDo == null || TriggerNodeToDo.IsTriggered)
{
if (NodeToDo.IsParallelAgentShareEditor)
{
Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor";
}
else
{
Procedure = "GUBP" + ProcedureInfix + "_UAT_Node";
if (!NodeToDo.IsSticky)
{
Procedure += "_Parallel";
}
if (NodeToDo.AgentSharingGroup != "")
{
Procedure += "_AgentShare";
}
}
if (NodeToDo.IsSticky && NodeToDo == LastSticky)
{
Procedure += "_Release";
}
}
else
{
if(TriggerNodeToDo.RequiresRecursiveWorkflow)
{
Procedure = "GUBP_UAT_Trigger"; //here we run a recursive workflow to wait for the trigger
}
else
{
Procedure = "GUBP_Hardcoded_Trigger"; //here we advance the state in the hardcoded workflow so folks can approve
}
Parameters.Add(new Tuple<string, string>("TriggerState", TriggerNodeToDo.StateName));
Parameters.Add(new Tuple<string, string>("ActionText", TriggerNodeToDo.ActionText));
Parameters.Add(new Tuple<string, string>("DescText", TriggerNodeToDo.DescriptionText));
if (NodeToDo.RecipientsForFailureEmails.Length > 0)
{
Parameters.Add(new Tuple<string, string>("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails)));
}
}
string ActualParameterArgs = String.Join(", ", Parameters.Select(x => String.Format("{{actualParameterName => '{0}', value => '{1}'}}", x.Item1, x.Item2)));
string Args = String.Format("{0}, subprocedure => '{1}', parallel => '{2}', jobStepName => '{3}', actualParameter => [{4}]", BaseArgs, Procedure, DoParallel? 1 : 0, NodeToDo.Name, ActualParameterArgs);
List<BuildNode> UncompletedEcDeps = new List<BuildNode>();
{
foreach (BuildNode Dep in NodeToDo.AllDirectDependencies)
{
if (!Dep.IsComplete && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC
{
if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(NodeToDo))
{
throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name);
}
UncompletedEcDeps.Add(Dep);
}
}
}
string PreCondition = GetPreConditionForNode(OrdereredToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps);
string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath);
string MyAgentGroup = NodeToDo.AgentSharingGroup;
bool bDoNestedJobstep = false;
bool bDoFirstNestedJobstep = false;
string NodeParentPath = ParentPath;
if (MyAgentGroup != "")
{
bDoNestedJobstep = true;
NodeParentPath = ParentPath + "/jobSteps[" + MyAgentGroup + "]";
List<BuildNode> MyChain = AgentGroupChains[MyAgentGroup];
int MyIndex = MyChain.IndexOf(NodeToDo);
if (MyIndex <= 0)
{
bDoFirstNestedJobstep = bDoNestedJobstep;
}
}
if (bDoNestedJobstep)
{
if (bDoFirstNestedJobstep)
{
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}', jobStepName => '{1}', parallel => '1'",
ParentPath, MyAgentGroup);
if (!String.IsNullOrEmpty(PreCondition))
{
NestArgs = NestArgs + ", precondition => " + PreCondition;
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetPool', subprocedure => 'GUBP{3}_AgentShare_GetPool', parallel => '1', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{4}'}}, {{actualParameterName => 'NodeName', value => '{5}'}}]",
ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, MyAgentGroup, NodeToDo.Name);
if (!String.IsNullOrEmpty(PreCondition))
{
NestArgs = NestArgs + ", precondition => " + PreCondition;
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetAgent', subprocedure => 'GUBP{3}_AgentShare_GetAgent', parallel => '1', exclusiveMode => 'call', resourceName => '{4}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{5}'}}, {{actualParameterName => 'NodeName', value=> '{6}'}}]",
ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix,
String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", MyAgentGroup),
MyAgentGroup, NodeToDo.Name);
{
NestArgs = NestArgs + ", precondition => ";
NestArgs = NestArgs + "\"\\$\" . \"[/javascript if(";
NestArgs = NestArgs + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetPool]/status') == 'completed'";
NestArgs = NestArgs + ") true;]\"";
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
PreCondition = "\"\\$\" . \"[/javascript if(";
PreCondition = PreCondition + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetAgent]/status') == 'completed'";
PreCondition = PreCondition + ") true;]\"";
}
}
Args = Args.Replace(String.Format("parentPath => '{0}'", ParentPath), String.Format("parentPath => '{0}'", NodeParentPath));
Args = Args.Replace("UAT_Node_Parallel_AgentShare", "UAT_Node_Parallel_AgentShare3");
}
if (!String.IsNullOrEmpty(PreCondition))
{
Args = Args + ", precondition => " + PreCondition;
}
if (!String.IsNullOrEmpty(RunCondition))
{
Args = Args + ", condition => " + RunCondition;
}
#if false
// this doesn't work because it includes precondition time
if (GUBPNodes[NodeToDo].TimeoutInMinutes() > 0)
{
Args = Args + String.Format(" --timeLimitUnits minutes --timeLimit {0}", GUBPNodes[NodeToDo].TimeoutInMinutes());
}
#endif
if (Sticky && NodeToDo == LastSticky)
{
Args = Args + ", releaseMode => 'release'";
}
Args = Args + "});";
StepList.Add(Args);
if (MyAgentGroup != "" && !bDoNestedJobstep)
{
List<BuildNode> MyChain = AgentGroupChains[MyAgentGroup];
int MyIndex = MyChain.IndexOf(NodeToDo);
if (MyIndex == MyChain.Count - 1)
{
string RelPreCondition = "\"\\$\" . \"[/javascript if(";
// this runs "parallel", but we a precondition to serialize it
RelPreCondition = RelPreCondition + "getProperty('" + ParentPath + "/jobSteps[" + NodeToDo.Name + "]/status') == 'completed'";
RelPreCondition = RelPreCondition + ") true;]\"";
// we need to release the resource
string RelArgs = String.Format("{0}, subprocedure => 'GUBP_Release_AgentShare', parallel => '1', jobStepName => 'Release_{1}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', valued => '{2}'}}], releaseMode => 'release', precondition => '{3}'",
BaseArgs, MyAgentGroup, MyAgentGroup, RelPreCondition);
StepList.Add(RelArgs);
}
}
}
}
WriteECPerl(StepList);
}
bool bHasTests = OrdereredToDo.Any(x => x.Node.IsTest());
RunECTool(String.Format("setProperty \"/myWorkflow/HasTests\" \"{0}\"", bHasTests));
{
ECProps.Add("GUBP_LoadedProps=1");
string BranchDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchDef.properties");
CommandUtils.WriteAllLines(BranchDefFile, ECProps.ToArray());
RunECTool(String.Format("setProperty \"/myWorkflow/BranchDefFile\" \"{0}\"", BranchDefFile.Replace("\\", "\\\\")));
}
{
ECProps.Add("GUBP_LoadedJobProps=1");
string BranchJobDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchJobDef.properties");
CommandUtils.WriteAllLines(BranchJobDefFile, ECProps.ToArray());
RunECTool(String.Format("setProperty \"/myJob/BranchJobDefFile\" \"{0}\"", BranchJobDefFile.Replace("\\", "\\\\")));
}
}
/// <summary>
/// Main method.
/// </summary>
/// <param name="CommandLine">Command line</param>
public static void Process(string[] CommandLine)
{
// Initial check for local or build machine runs BEFORE we parse the command line (We need this value set
// in case something throws the exception while parsing the command line)
IsBuildMachine = !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("uebp_LOCAL_ROOT"));
// Scan the command line for commands to execute.
var CommandsToExecute = new List<CommandInfo>();
var AdditionalScriptsFolders = new List<string>();
ParseCommandLine(CommandLine, CommandsToExecute, AdditionalScriptsFolders);
// Check for build machine override (force local)
IsBuildMachine = GlobalCommandLine.ForceLocal ? false : IsBuildMachine;
Log.TraceInformation("IsBuildMachine={0}", IsBuildMachine);
Environment.SetEnvironmentVariable("IsBuildMachine", IsBuildMachine ? "1" : "0");
// should we kill processes on exit
ShouldKillProcesses = !GlobalCommandLine.NoKill;
Log.TraceInformation("ShouldKillProcesses={0}", ShouldKillProcesses);
if (CommandsToExecute.Count == 0 && GlobalCommandLine.Help)
{
DisplayHelp();
return;
}
// Disable AutoSDKs if specified on the command line
if (GlobalCommandLine.NoAutoSDK)
{
UEBuildPlatform.bAllowAutoSDKSwitching = false;
}
// Setup environment
Log.TraceInformation("Setting up command environment.");
CommandUtils.InitCommandEnvironment();
// Change CWD to UE4 root.
Environment.CurrentDirectory = CommandUtils.CmdEnv.LocalRoot;
// Fill in the project info
UnrealBuildTool.UProjectInfo.FillProjectInfo();
// Clean rules folders up
ProjectUtils.CleanupFolders();
// Compile scripts.
Log.TraceInformation("Compiling scripts.");
ScriptCompiler Compiler = new ScriptCompiler();
using(CommandUtils.TelemetryStopwatch ScriptCompileStopwatch = new CommandUtils.TelemetryStopwatch("ScriptCompile"))
{
Compiler.FindAndCompileAllScripts(AdditionalScriptsFolders: AdditionalScriptsFolders);
}
if (GlobalCommandLine.CompileOnly)
{
Log.TraceInformation("Compilation successful, exiting (CompileOnly)");
return;
}
if (GlobalCommandLine.List)
{
ListAvailableCommands(Compiler.Commands);
return;
}
if (GlobalCommandLine.Help)
{
DisplayHelp(CommandsToExecute, Compiler.Commands);
return;
}
// Enable or disable P4 support
CommandUtils.InitP4Support(CommandsToExecute, Compiler.Commands);
if (CommandUtils.P4Enabled)
{
Log.TraceInformation("Setting up Perforce environment.");
CommandUtils.InitP4Environment();
CommandUtils.InitDefaultP4Connection();
}
// Find and execute commands.
Execute(CommandsToExecute, Compiler.Commands);
return;
}
