public void DoCommanderSetup(IEnumerable <BuildNode> AllNodes, IEnumerable <AggregateNode> AllAggregates, List <BuildNode> OrderedToDo, List <BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, TriggerNode ExplicitTrigger, List <TriggerNode> UnfinishedTriggers)
{
// Check there's some nodes in the graph
if (OrderedToDo.Count == 0)
{
throw new AutomationException("No nodes to do!");
}
// Make sure everything before the explicit trigger is completed.
if (ExplicitTrigger != null)
{
foreach (BuildNode NodeToDo in OrderedToDo)
{
if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger)) // if something is already finished, we don't put it into EC
{
throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name);
}
}
}
// Update all the EC properties, and the branch definition files
WriteProperties(AllNodes, AllAggregates, OrderedToDo, SortedNodes, TimeIndex, TimeQuantum);
// Write all the job setup
WriteJobSteps(OrderedToDo, bSkipTriggers);
// Export it as JSON too
WriteJson(OrderedToDo, bSkipTriggers);
}
public void DoCommanderSetup(IEnumerable <BuildNode> AllNodes, IEnumerable <AggregateNode> AllAggregates, List <BuildNode> OrdereredToDo, List <BuildNode> SortedNodes, int TimeIndex, int TimeQuantum, bool bSkipTriggers, bool bFake, bool bFakeEC, string CLString, TriggerNode ExplicitTrigger, List <TriggerNode> UnfinishedTriggers, string FakeFail)
{
List <AggregateNode> SeparatePromotables = FindPromotables(AllAggregates);
Dictionary <BuildNode, List <AggregateNode> > DependentPromotions = FindDependentPromotables(AllNodes, SeparatePromotables);
Dictionary <BuildNode, int> FullNodeListSortKey = GetDisplayOrder(SortedNodes);
if (OrdereredToDo.Count == 0)
{
throw new AutomationException("No nodes to do!");
}
List <string> ECProps = new List <string>();
ECProps.Add(String.Format("TimeIndex={0}", TimeIndex));
foreach (BuildNode Node in SortedNodes.Where(x => x.FrequencyShift != BuildNode.ExplicitFrequencyShift))
{
ECProps.Add(string.Format("AllNodes/{0}={1}", Node.Name, GetNodeForAllNodesProperty(Node, TimeQuantum)));
}
foreach (KeyValuePair <BuildNode, int> NodePair in FullNodeListSortKey.Where(x => x.Key.FrequencyShift != BuildNode.ExplicitFrequencyShift))
{
ECProps.Add(string.Format("SortKey/{0}={1}", NodePair.Key.Name, NodePair.Value));
}
foreach (KeyValuePair <BuildNode, List <AggregateNode> > NodePair in DependentPromotions)
{
ECProps.Add(string.Format("DependentPromotions/{0}={1}", NodePair.Key.Name, String.Join(" ", NodePair.Value.Select(x => x.Name))));
}
foreach (AggregateNode Node in SeparatePromotables)
{
ECProps.Add(string.Format("PossiblePromotables/{0}={1}", Node.Name, ""));
}
List <string> ECJobProps = new List <string>();
if (ExplicitTrigger != null)
{
ECJobProps.Add("IsRoot=0");
}
else
{
ECJobProps.Add("IsRoot=1");
}
// here we are just making sure everything before the explicit trigger is completed.
if (ExplicitTrigger != null)
{
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete && NodeToDo != ExplicitTrigger && !NodeToDo.DependsOn(ExplicitTrigger)) // if something is already finished, we don't put it into EC
{
throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo.Name, ExplicitTrigger.Name);
}
}
}
BuildNode LastSticky = null;
bool HitNonSticky = false;
bool bHaveECNodes = false;
List <string> StepList = new List <string>();
StepList.Add("use strict;");
StepList.Add("use diagnostics;");
StepList.Add("use ElectricCommander();");
StepList.Add("my $ec = new ElectricCommander;");
StepList.Add("$ec->setTimeout(600);");
StepList.Add("my $batch = $ec->newBatch(\"serial\");");
// sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
bHaveECNodes = true;
if (NodeToDo.IsSticky)
{
LastSticky = NodeToDo;
if (HitNonSticky && !bSkipTriggers)
{
throw new AutomationException("Sticky and non-sticky jobs did not sort right.");
}
}
else
{
HitNonSticky = true;
}
}
}
using (CommandUtils.TelemetryStopwatch PerlOutputStopwatch = new CommandUtils.TelemetryStopwatch("PerlOutput"))
{
string ParentPath = Command.ParseParamValue("ParentPath");
string BaseArgs = String.Format("$batch->createJobStep({{parentPath => '{0}'", ParentPath);
bool bHasNoop = false;
if (LastSticky == null && bHaveECNodes)
{
// if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource
string Args = String.Format("{0}, subprocedure => 'GUBP_UAT_Node', parallel => '0', jobStepName => 'Noop', actualParameter => [{{actualParameterName => 'NodeName', value => 'Noop'}}, {{actualParameterName => 'Sticky', value =>'1' }}], releaseMode => 'release'}});", BaseArgs);
StepList.Add(Args);
bHasNoop = true;
}
Dictionary <string, List <BuildNode> > AgentGroupChains = new Dictionary <string, List <BuildNode> >();
List <BuildNode> StickyChain = new List <BuildNode>();
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
string MyAgentGroup = NodeToDo.AgentSharingGroup;
if (MyAgentGroup != "")
{
if (!AgentGroupChains.ContainsKey(MyAgentGroup))
{
AgentGroupChains.Add(MyAgentGroup, new List <BuildNode> {
NodeToDo
});
}
else
{
AgentGroupChains[MyAgentGroup].Add(NodeToDo);
}
}
}
if (NodeToDo.IsSticky)
{
if (!StickyChain.Contains(NodeToDo))
{
StickyChain.Add(NodeToDo);
}
}
}
foreach (BuildNode NodeToDo in OrdereredToDo)
{
if (!NodeToDo.IsComplete) // if something is already finished, we don't put it into EC
{
List <string> NodeProps = GetECPropsForNode(NodeToDo);
ECProps.AddRange(NodeProps);
bool Sticky = NodeToDo.IsSticky;
if (NodeToDo.IsSticky)
{
if (NodeToDo.AgentSharingGroup != "")
{
throw new AutomationException("Node {0} is both agent sharing and sitcky.", NodeToDo.Name);
}
if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
{
throw new AutomationException("Node {0} is sticky, but {1} hosted. Sticky nodes must be PC hosted.", NodeToDo.Name, NodeToDo.AgentPlatform);
}
if (NodeToDo.AgentRequirements != "")
{
throw new AutomationException("Node {0} is sticky but has agent requirements.", NodeToDo.Name);
}
}
string ProcedureInfix = "";
if (NodeToDo.AgentPlatform != UnrealTargetPlatform.Unknown && NodeToDo.AgentPlatform != UnrealTargetPlatform.Win64)
{
ProcedureInfix = "_" + NodeToDo.AgentPlatform.ToString();
}
bool DoParallel = !Sticky || NodeToDo.IsParallelAgentShareEditor;
TriggerNode TriggerNodeToDo = NodeToDo as TriggerNode;
List <Tuple <string, string> > Parameters = new List <Tuple <string, string> >();
Parameters.Add(new Tuple <string, string>("NodeName", NodeToDo.Name));
Parameters.Add(new Tuple <string, string>("Sticky", NodeToDo.IsSticky ? "1" : "0"));
if (NodeToDo.AgentSharingGroup != "")
{
Parameters.Add(new Tuple <string, string>("AgentSharingGroup", NodeToDo.AgentSharingGroup));
}
string Procedure;
if (TriggerNodeToDo == null || TriggerNodeToDo.IsTriggered)
{
if (NodeToDo.IsParallelAgentShareEditor)
{
Procedure = "GUBP_UAT_Node_Parallel_AgentShare_Editor";
}
else
{
Procedure = "GUBP" + ProcedureInfix + "_UAT_Node";
if (!NodeToDo.IsSticky)
{
Procedure += "_Parallel";
}
if (NodeToDo.AgentSharingGroup != "")
{
Procedure += "_AgentShare";
}
}
if (NodeToDo.IsSticky && NodeToDo == LastSticky)
{
Procedure += "_Release";
}
}
else
{
if (TriggerNodeToDo.RequiresRecursiveWorkflow)
{
Procedure = "GUBP_UAT_Trigger"; //here we run a recursive workflow to wait for the trigger
}
else
{
Procedure = "GUBP_Hardcoded_Trigger"; //here we advance the state in the hardcoded workflow so folks can approve
}
Parameters.Add(new Tuple <string, string>("TriggerState", TriggerNodeToDo.StateName));
Parameters.Add(new Tuple <string, string>("ActionText", TriggerNodeToDo.ActionText));
Parameters.Add(new Tuple <string, string>("DescText", TriggerNodeToDo.DescriptionText));
if (NodeToDo.RecipientsForFailureEmails.Length > 0)
{
Parameters.Add(new Tuple <string, string>("EmailsForTrigger", String.Join(" ", NodeToDo.RecipientsForFailureEmails)));
}
}
string ActualParameterArgs = String.Join(", ", Parameters.Select(x => String.Format("{{actualParameterName => '{0}', value => '{1}'}}", x.Item1, x.Item2)));
string Args = String.Format("{0}, subprocedure => '{1}', parallel => '{2}', jobStepName => '{3}', actualParameter => [{4}]", BaseArgs, Procedure, DoParallel? 1 : 0, NodeToDo.Name, ActualParameterArgs);
List <BuildNode> UncompletedEcDeps = new List <BuildNode>();
{
foreach (BuildNode Dep in NodeToDo.AllDirectDependencies)
{
if (!Dep.IsComplete && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC
{
if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(NodeToDo))
{
throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo.Name, Dep.Name);
}
UncompletedEcDeps.Add(Dep);
}
}
}
string PreCondition = GetPreConditionForNode(OrdereredToDo, ParentPath, bHasNoop, AgentGroupChains, StickyChain, NodeToDo, UncompletedEcDeps);
string RunCondition = GetRunConditionForNode(UncompletedEcDeps, ParentPath);
string MyAgentGroup = NodeToDo.AgentSharingGroup;
bool bDoNestedJobstep = false;
bool bDoFirstNestedJobstep = false;
string NodeParentPath = ParentPath;
if (MyAgentGroup != "")
{
bDoNestedJobstep = true;
NodeParentPath = ParentPath + "/jobSteps[" + MyAgentGroup + "]";
List <BuildNode> MyChain = AgentGroupChains[MyAgentGroup];
int MyIndex = MyChain.IndexOf(NodeToDo);
if (MyIndex <= 0)
{
bDoFirstNestedJobstep = bDoNestedJobstep;
}
}
if (bDoNestedJobstep)
{
if (bDoFirstNestedJobstep)
{
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}', jobStepName => '{1}', parallel => '1'",
ParentPath, MyAgentGroup);
if (!String.IsNullOrEmpty(PreCondition))
{
NestArgs = NestArgs + ", precondition => " + PreCondition;
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetPool', subprocedure => 'GUBP{3}_AgentShare_GetPool', parallel => '1', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{4}'}}, {{actualParameterName => 'NodeName', value => '{5}'}}]",
ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix, MyAgentGroup, NodeToDo.Name);
if (!String.IsNullOrEmpty(PreCondition))
{
NestArgs = NestArgs + ", precondition => " + PreCondition;
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetAgent', subprocedure => 'GUBP{3}_AgentShare_GetAgent', parallel => '1', exclusiveMode => 'call', resourceName => '{4}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{5}'}}, {{actualParameterName => 'NodeName', value=> '{6}'}}]",
ParentPath, MyAgentGroup, MyAgentGroup, ProcedureInfix,
String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", MyAgentGroup),
MyAgentGroup, NodeToDo.Name);
{
NestArgs = NestArgs + ", precondition => ";
NestArgs = NestArgs + "\"\\$\" . \"[/javascript if(";
NestArgs = NestArgs + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetPool]/status') == 'completed'";
NestArgs = NestArgs + ") true;]\"";
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
PreCondition = "\"\\$\" . \"[/javascript if(";
PreCondition = PreCondition + "getProperty('" + ParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetAgent]/status') == 'completed'";
PreCondition = PreCondition + ") true;]\"";
}
}
Args = Args.Replace(String.Format("parentPath => '{0}'", ParentPath), String.Format("parentPath => '{0}'", NodeParentPath));
Args = Args.Replace("UAT_Node_Parallel_AgentShare", "UAT_Node_Parallel_AgentShare3");
}
if (!String.IsNullOrEmpty(PreCondition))
{
Args = Args + ", precondition => " + PreCondition;
}
if (!String.IsNullOrEmpty(RunCondition))
{
Args = Args + ", condition => " + RunCondition;
}
#if false
// this doesn't work because it includes precondition time
if (GUBPNodes[NodeToDo].TimeoutInMinutes() > 0)
{
Args = Args + String.Format(" --timeLimitUnits minutes --timeLimit {0}", GUBPNodes[NodeToDo].TimeoutInMinutes());
}
#endif
if (Sticky && NodeToDo == LastSticky)
{
Args = Args + ", releaseMode => 'release'";
}
Args = Args + "});";
StepList.Add(Args);
if (MyAgentGroup != "" && !bDoNestedJobstep)
{
List <BuildNode> MyChain = AgentGroupChains[MyAgentGroup];
int MyIndex = MyChain.IndexOf(NodeToDo);
if (MyIndex == MyChain.Count - 1)
{
string RelPreCondition = "\"\\$\" . \"[/javascript if(";
// this runs "parallel", but we a precondition to serialize it
RelPreCondition = RelPreCondition + "getProperty('" + ParentPath + "/jobSteps[" + NodeToDo.Name + "]/status') == 'completed'";
RelPreCondition = RelPreCondition + ") true;]\"";
// we need to release the resource
string RelArgs = String.Format("{0}, subprocedure => 'GUBP_Release_AgentShare', parallel => '1', jobStepName => 'Release_{1}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', valued => '{2}'}}], releaseMode => 'release', precondition => '{3}'",
BaseArgs, MyAgentGroup, MyAgentGroup, RelPreCondition);
StepList.Add(RelArgs);
}
}
}
}
WriteECPerl(StepList);
}
bool bHasTests = OrdereredToDo.Any(x => x.Node.IsTest());
RunECTool(String.Format("setProperty \"/myWorkflow/HasTests\" \"{0}\"", bHasTests));
{
ECProps.Add("GUBP_LoadedProps=1");
string BranchDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchDef.properties");
CommandUtils.WriteAllLines(BranchDefFile, ECProps.ToArray());
RunECTool(String.Format("setProperty \"/myWorkflow/BranchDefFile\" \"{0}\"", BranchDefFile.Replace("\\", "\\\\")));
}
{
ECProps.Add("GUBP_LoadedJobProps=1");
string BranchJobDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchJobDef.properties");
CommandUtils.WriteAllLines(BranchJobDefFile, ECProps.ToArray());
RunECTool(String.Format("setProperty \"/myJob/BranchJobDefFile\" \"{0}\"", BranchJobDefFile.Replace("\\", "\\\\")));
}
}