public GUBPBranchConfig(IEnumerable<UnrealTargetPlatform> InHostPlatforms, string InBranchName, BranchInfo InBranch, GUBPBranchHacker.BranchOptions InBranchOptions, bool bInForceIncrementalCompile, JobInfo JobInfo)
{
HostPlatforms = new List<UnrealTargetPlatform>(InHostPlatforms);
BranchName = InBranchName;
Branch = InBranch;
BranchOptions = InBranchOptions;
bForceIncrementalCompile = bInForceIncrementalCompile;
this.JobInfo = JobInfo;
}
public GUBPBranchConfig(int InCL, IEnumerable<UnrealTargetPlatform> InHostPlatforms, string InBranchName, BranchInfo InBranch, GUBPBranchHacker.BranchOptions InBranchOptions, bool bInForceIncrementalCompile, bool bInPreflightBuild, string InPreflightMangleSuffix)
{
CL = InCL;
HostPlatforms = new List<UnrealTargetPlatform>(InHostPlatforms);
BranchName = InBranchName;
Branch = InBranch;
BranchOptions = InBranchOptions;
bForceIncrementalCompile = bInForceIncrementalCompile;
bPreflightBuild = bInPreflightBuild;
PreflightMangleSuffix = InPreflightMangleSuffix;
}
private GUBPBranchHacker.BranchOptions GetBranchOptions(string Branch)
{
if (BranchHackers == null)
{
BranchHackers = new List<GUBPBranchHacker>();
Assembly[] LoadedAssemblies = AppDomain.CurrentDomain.GetAssemblies();
foreach (Assembly Dll in LoadedAssemblies)
{
Type[] AllTypes = GetTypesFromAssembly(Dll);
foreach (var PotentialConfigType in AllTypes)
{
if (PotentialConfigType != typeof(GUBPBranchHacker) && typeof(GUBPBranchHacker).IsAssignableFrom(PotentialConfigType))
{
GUBPBranchHacker Config = Activator.CreateInstance(PotentialConfigType) as GUBPBranchHacker;
if (Config != null)
{
BranchHackers.Add(Config);
}
}
}
}
}
GUBPBranchHacker.BranchOptions Result = new GUBPBranchHacker.BranchOptions();
foreach (GUBPBranchHacker Hacker in BranchHackers)
{
Hacker.ModifyOptions(this, ref Result, Branch);
}
return Result;
}
public override void ExecuteBuild()
{
Log("************************* GUBP");
string PreflightShelveCLString = GetEnvVar("uebp_PreflightShelveCL");
if ((!String.IsNullOrEmpty(PreflightShelveCLString) && IsBuildMachine) || ParseParam("PreflightTest"))
{
Log("**** Preflight shelve {0}", PreflightShelveCLString);
if (!String.IsNullOrEmpty(PreflightShelveCLString))
{
PreflightShelveCL = int.Parse(PreflightShelveCLString);
if (PreflightShelveCL < 2000000)
{
throw new AutomationException(String.Format( "{0} does not look like a CL", PreflightShelveCL));
}
}
bPreflightBuild = true;
}
ECProject = ParseParamValue("ECProject");
if (ECProject == null)
{
ECProject = "";
}
HostPlatforms = new List<UnrealTargetPlatform>();
if (!ParseParam("NoPC"))
{
HostPlatforms.Add(UnrealTargetPlatform.Win64);
}
if (P4Enabled)
{
BranchName = P4Env.BuildRootP4;
}
else
{
BranchName = ParseParamValue("BranchName", "");
}
BranchOptions = GetBranchOptions(BranchName);
bool WithMac = !BranchOptions.PlatformsToRemove.Contains(UnrealTargetPlatform.Mac);
if (ParseParam("NoMac"))
{
WithMac = false;
}
if (WithMac)
{
HostPlatforms.Add(UnrealTargetPlatform.Mac);
}
bool WithLinux = !BranchOptions.PlatformsToRemove.Contains(UnrealTargetPlatform.Linux);
bool WithoutLinux = ParseParam("NoLinux");
// @TODO: exclude temporarily unless running on a Linux machine to prevent spurious GUBP failures
if (UnrealBuildTool.BuildHostPlatform.Current.Platform != UnrealTargetPlatform.Linux || ParseParam("NoLinux"))
{
WithLinux = false;
}
if (WithLinux)
{
HostPlatforms.Add(UnrealTargetPlatform.Linux);
}
bForceIncrementalCompile = ParseParam("ForceIncrementalCompile");
bool bNoAutomatedTesting = ParseParam("NoAutomatedTesting") || BranchOptions.bNoAutomatedTesting;
StoreName = ParseParamValue("Store");
string StoreSuffix = ParseParamValue("StoreSuffix", "");
if (bPreflightBuild)
{
int PreflightUID = ParseParamInt("PreflightUID", 0);
PreflightMangleSuffix = String.Format("-PF-{0}-{1}", PreflightShelveCL, PreflightUID);
StoreSuffix = StoreSuffix + PreflightMangleSuffix;
}
CL = ParseParamInt("CL", 0);
bool bCleanLocalTempStorage = ParseParam("CleanLocal");
bool bChanges = ParseParam("Changes") || ParseParam("AllChanges");
bool bHistory = ParseParam("History") || bChanges;
bool bListOnly = ParseParam("ListOnly") || bHistory;
bool bSkipTriggers = ParseParam("SkipTriggers");
bFake = ParseParam("fake");
bool bFakeEC = ParseParam("FakeEC");
TimeIndex = ParseParamInt("TimeIndex", 0);
if (TimeIndex == 0)
{
TimeIndex = ParseParamInt("UserTimeIndex", 0);
}
bNoIOSOnPC = HostPlatforms.Contains(UnrealTargetPlatform.Mac);
bool bSaveSharedTempStorage = false;
if (bHistory && !P4Enabled)
{
throw new AutomationException("-Changes and -History require -P4.");
}
bool LocalOnly = true;
string CLString = "";
if (String.IsNullOrEmpty(StoreName))
{
if (P4Enabled)
{
if (CL == 0)
{
CL = P4Env.Changelist;
}
CLString = String.Format("{0}", CL);
StoreName = P4Env.BuildRootEscaped + "-" + CLString;
bSaveSharedTempStorage = CommandUtils.IsBuildMachine;
LocalOnly = false;
}
else
{
StoreName = "TempLocal";
bSaveSharedTempStorage = false;
}
}
StoreName = StoreName + StoreSuffix;
if (bFakeEC)
{
LocalOnly = true;
}
if (bSaveSharedTempStorage)
{
if (!HaveSharedTempStorage(true))
{
throw new AutomationException("Request to save to temp storage, but {0} is unavailable.", UE4TempStorageDirectory());
}
bSignBuildProducts = true;
}
else if (!LocalOnly && !HaveSharedTempStorage(false))
{
Log("Looks like we want to use shared temp storage, but since we don't have it, we won't use it.");
LocalOnly = true;
}
bool CommanderSetup = ParseParam("CommanderJobSetupOnly");
string ExplicitTrigger = "";
if (CommanderSetup)
{
ExplicitTrigger = ParseParamValue("TriggerNode");
if (ExplicitTrigger == null)
{
ExplicitTrigger = "";
}
}
if (ParseParam("CIS") && ExplicitTrigger == "" && CommanderSetup) // explicit triggers will already have a time index assigned
{
if (!P4Enabled)
{
throw new AutomationException("Can't have -CIS without P4 support");
}
var P4IndexFileP4 = CombinePaths(PathSeparator.Slash, CommandUtils.P4Env.BuildRootP4, "Engine", "Build", "CISCounter.txt");
var P4IndexFileLocal = CombinePaths(CmdEnv.LocalRoot, "Engine", "Build", "CISCounter.txt");
int Retry = 0;
bool bDone = false;
while (++Retry < 20 && !bDone)
{
int NowMinutes = (int)((DateTime.UtcNow - new DateTime(2014, 1, 1)).TotalMinutes);
if (NowMinutes < 3 * 30 * 24)
{
throw new AutomationException("bad date calc");
}
if (!FileExists_NoExceptions(P4IndexFileLocal))
{
Log("{0} doesn't exist, checking in a new one", P4IndexFileP4);
WriteAllText(P4IndexFileLocal, "-1 0");
int WorkingCL = -1;
try
{
WorkingCL = P4.CreateChange(P4Env.Client, "Adding new CIS Counter");
P4.Add(WorkingCL, P4IndexFileP4);
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL);
}
catch (Exception)
{
Log("Add of CIS counter failed, assuming it now exists.");
if (WorkingCL > 0)
{
P4.DeleteChange(WorkingCL);
}
}
}
P4.Sync("-f " + P4IndexFileP4 + "#head");
if (!FileExists_NoExceptions(P4IndexFileLocal))
{
Log("{0} doesn't exist, checking in a new one", P4IndexFileP4);
WriteAllText(P4IndexFileLocal, "-1 0");
int WorkingCL = -1;
try
{
WorkingCL = P4.CreateChange(P4Env.Client, "Adding new CIS Counter");
P4.Add(WorkingCL, P4IndexFileP4);
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL);
}
catch (Exception)
{
Log("Add of CIS counter failed, assuming it now exists.");
if (WorkingCL > 0)
{
P4.DeleteChange(WorkingCL);
}
}
}
var Data = ReadAllText(P4IndexFileLocal);
var Parts = Data.Split(" ".ToCharArray());
int Index = int.Parse(Parts[0]);
int Minutes = int.Parse(Parts[1]);
int DeltaMinutes = NowMinutes - Minutes;
int TimeQuantum = 20;
if(BranchOptions.QuantumOverride != 0)
{
TimeQuantum = BranchOptions.QuantumOverride;
}
int NewIndex = Index + 1;
if (DeltaMinutes > TimeQuantum * 2)
{
if (DeltaMinutes > TimeQuantum * (1 << 8))
{
// it has been forever, lets just start over
NewIndex = 0;
}
else
{
int WorkingIndex = NewIndex + 1;
for (int WorkingDelta = DeltaMinutes - TimeQuantum; WorkingDelta > 0; WorkingDelta -= TimeQuantum, WorkingIndex++)
{
if (CountZeros(NewIndex) < CountZeros(WorkingIndex))
{
NewIndex = WorkingIndex;
}
}
}
}
{
var Line = String.Format("{0} {1}", NewIndex, NowMinutes);
Log("Attempting to write {0} with {1}", P4IndexFileP4, Line);
int WorkingCL = -1;
try
{
WorkingCL = P4.CreateChange(P4Env.Client, "Updating CIS Counter");
P4.Edit(WorkingCL, P4IndexFileP4);
WriteAllText(P4IndexFileLocal, Line);
int SubmittedCL;
P4.Submit(WorkingCL, out SubmittedCL);
bDone = true;
TimeIndex = NewIndex;
}
catch (Exception)
{
Log("Edit of CIS counter failed, assuming someone else checked in, retrying.");
if (WorkingCL > 0)
{
P4.DeleteChange(WorkingCL);
}
System.Threading.Thread.Sleep(30000);
}
}
}
if (!bDone)
{
throw new AutomationException("Failed to update the CIS counter after 20 tries.");
}
Log("Setting TimeIndex to {0}", TimeIndex);
}
Log("************************* CL: {0}", CL);
Log("************************* P4Enabled: {0}", P4Enabled);
foreach (var HostPlatform in HostPlatforms)
{
Log("************************* HostPlatform: {0}", HostPlatform.ToString());
}
Log("************************* StoreName: {0}", StoreName.ToString());
Log("************************* bCleanLocalTempStorage: {0}", bCleanLocalTempStorage);
Log("************************* bSkipTriggers: {0}", bSkipTriggers);
Log("************************* bSaveSharedTempStorage: {0}", bSaveSharedTempStorage);
Log("************************* bSignBuildProducts: {0}", bSignBuildProducts);
Log("************************* bFake: {0}", bFake);
Log("************************* bFakeEC: {0}", bFakeEC);
Log("************************* bHistory: {0}", bHistory);
Log("************************* TimeIndex: {0}", TimeIndex);
GUBPNodes = new Dictionary<string, GUBPNode>();
Branch = new BranchInfo(HostPlatforms);
if (IsBuildMachine || ParseParam("AllPlatforms"))
{
ActivePlatforms = new List<UnrealTargetPlatform>();
List<BranchInfo.BranchUProject> BranchCodeProjects = new List<BranchInfo.BranchUProject>();
BranchCodeProjects.Add(Branch.BaseEngineProject);
BranchCodeProjects.AddRange(Branch.CodeProjects);
BranchCodeProjects.RemoveAll(Project => BranchOptions.ExcludeNodes.Contains(Project.GameName));
foreach (var GameProj in BranchCodeProjects)
{
foreach (var Kind in BranchInfo.MonolithicKinds)
{
if (GameProj.Properties.Targets.ContainsKey(Kind))
{
var Target = GameProj.Properties.Targets[Kind];
foreach (var HostPlatform in HostPlatforms)
{
var Platforms = Target.Rules.GUBP_GetPlatforms_MonolithicOnly(HostPlatform);
var AdditionalPlatforms = Target.Rules.GUBP_GetBuildOnlyPlatforms_MonolithicOnly(HostPlatform);
var AllPlatforms = Platforms.Union(AdditionalPlatforms);
foreach (var Plat in AllPlatforms)
{
if (Target.Rules.SupportsPlatform(Plat) && !ActivePlatforms.Contains(Plat))
{
ActivePlatforms.Add(Plat);
}
}
}
}
}
}
}
else
{
ActivePlatforms = new List<UnrealTargetPlatform>(CommandUtils.KnownTargetPlatforms);
}
var SupportedPlatforms = new List<UnrealTargetPlatform>();
foreach(var Plat in ActivePlatforms)
{
if(!BranchOptions.PlatformsToRemove.Contains(Plat))
{
SupportedPlatforms.Add(Plat);
}
}
ActivePlatforms = SupportedPlatforms;
foreach (var Plat in ActivePlatforms)
{
Log("Active Platform: {0}", Plat.ToString());
}
if (HostPlatforms.Count >= 2)
{
// make sure each project is set up with the right assumptions on monolithics that prefer a platform.
foreach (var CodeProj in Branch.CodeProjects)
{
var OptionsMac = CodeProj.Options(UnrealTargetPlatform.Mac);
var OptionsPC = CodeProj.Options(UnrealTargetPlatform.Win64);
var MacMonos = GetMonolithicPlatformsForUProject(UnrealTargetPlatform.Mac, CodeProj, false);
var PCMonos = GetMonolithicPlatformsForUProject(UnrealTargetPlatform.Win64, CodeProj, false);
if (!OptionsMac.bIsPromotable && OptionsPC.bIsPromotable &&
(MacMonos.Contains(UnrealTargetPlatform.IOS) || PCMonos.Contains(UnrealTargetPlatform.IOS)))
{
throw new AutomationException("Project {0} is promotable for PC, not promotable for Mac and uses IOS monothics. Since Mac is the preferred platform for IOS, please add Mac as a promotable platform.", CodeProj.GameName);
}
if (OptionsMac.bIsPromotable && !OptionsPC.bIsPromotable &&
(MacMonos.Contains(UnrealTargetPlatform.Android) || PCMonos.Contains(UnrealTargetPlatform.Android)))
{
throw new AutomationException("Project {0} is not promotable for PC, promotable for Mac and uses Android monothics. Since PC is the preferred platform for Android, please add PC as a promotable platform.", CodeProj.GameName);
}
}
}
AddNode(new VersionFilesNode());
foreach (var HostPlatform in HostPlatforms)
{
AddNode(new ToolsForCompileNode(HostPlatform));
if (!BranchOptions.ExcludePlatformsForEditor.Contains(HostPlatform))
{
AddNode(new RootEditorNode(HostPlatform));
AddNode(new ToolsNode(HostPlatform));
AddNode(new InternalToolsNode(HostPlatform));
if (HostPlatform == UnrealTargetPlatform.Win64 && ActivePlatforms.Contains(UnrealTargetPlatform.Linux))
{
if (!BranchOptions.ExcludePlatformsForEditor.Contains(UnrealTargetPlatform.Linux))
{
AddNode(new ToolsCrossCompileNode(HostPlatform));
}
}
foreach (var ProgramTarget in Branch.BaseEngineProject.Properties.Programs)
{
bool bInternalOnly;
bool SeparateNode;
bool CrossCompile;
if (ProgramTarget.Rules.GUBP_AlwaysBuildWithTools(HostPlatform, out bInternalOnly, out SeparateNode, out CrossCompile) && ProgramTarget.Rules.SupportsPlatform(HostPlatform) && SeparateNode)
{
if (bInternalOnly)
{
AddNode(new SingleInternalToolsNode(HostPlatform, ProgramTarget));
}
else
{
AddNode(new SingleToolsNode(HostPlatform, ProgramTarget));
}
}
if (ProgramTarget.Rules.GUBP_IncludeNonUnityToolTest())
{
AddNode(new NonUnityToolNode(HostPlatform, ProgramTarget));
}
}
foreach(var CodeProj in Branch.CodeProjects)
{
foreach(var ProgramTarget in CodeProj.Properties.Programs)
{
bool bInternalNodeOnly;
bool SeparateNode;
bool CrossCompile;
if(ProgramTarget.Rules.GUBP_AlwaysBuildWithTools(HostPlatform, out bInternalNodeOnly, out SeparateNode, out CrossCompile) && ProgramTarget.Rules.SupportsPlatform(HostPlatform) && SeparateNode)
{
if(bInternalNodeOnly)
{
AddNode(new SingleInternalToolsNode(HostPlatform, ProgramTarget));
}
else
{
AddNode(new SingleToolsNode(HostPlatform, ProgramTarget));
}
}
if(ProgramTarget.Rules.GUBP_IncludeNonUnityToolTest())
{
AddNode(new NonUnityToolNode(HostPlatform, ProgramTarget));
}
}
}
AddNode(new EditorAndToolsNode(this, HostPlatform));
if (bOrthogonalizeEditorPlatforms)
{
foreach (var Plat in ActivePlatforms)
{
if (Plat != HostPlatform && Plat != GetAltHostPlatform(HostPlatform))
{
if (Platform.Platforms[HostPlatform].CanHostPlatform(Plat))
{
AddNode(new EditorPlatformNode(HostPlatform, Plat));
}
}
}
}
}
bool DoASharedPromotable = false;
int NumSharedCode = 0;
foreach (var CodeProj in Branch.CodeProjects)
{
var Options = CodeProj.Options(HostPlatform);
if (Options.bIsPromotable && !Options.bSeparateGamePromotion)
{
NumSharedCode++;
}
}
var NonCodeProjectNames = new Dictionary<string, List<UnrealTargetPlatform>>();
var NonCodeFormalBuilds = new Dictionary<string, List<TargetRules.GUBPFormalBuild>>();
{
var Target = Branch.BaseEngineProject.Properties.Targets[TargetRules.TargetType.Editor];
foreach (var Codeless in Target.Rules.GUBP_NonCodeProjects_BaseEditorTypeOnly(HostPlatform))
{
var Proj = Branch.FindGame(Codeless.Key);
if (Proj == null)
{
Log(System.Diagnostics.TraceEventType.Information, "{0} was listed as a codeless project by GUBP_NonCodeProjects_BaseEditorTypeOnly, however it does not exist in this branch.", Codeless.Key);
}
else if (Proj.Properties.bIsCodeBasedProject)
{
if (!Branch.NonCodeProjects.Contains(Proj))
{
Branch.NonCodeProjects.Add(Proj);
NonCodeProjectNames.Add(Codeless.Key, Codeless.Value);
}
}
else
{
NonCodeProjectNames.Add(Codeless.Key, Codeless.Value);
}
}
var TempNonCodeFormalBuilds = Target.Rules.GUBP_GetNonCodeFormalBuilds_BaseEditorTypeOnly();
var HostMonos = GetMonolithicPlatformsForUProject(HostPlatform, Branch.BaseEngineProject, true);
foreach (var Codeless in TempNonCodeFormalBuilds)
{
if (NonCodeProjectNames.ContainsKey(Codeless.Key))
{
var PlatList = Codeless.Value;
var NewPlatList = new List<TargetRules.GUBPFormalBuild>();
foreach (var PlatPair in PlatList)
{
if (HostMonos.Contains(PlatPair.TargetPlatform))
{
NewPlatList.Add(PlatPair);
}
}
if (NewPlatList.Count > 0)
{
NonCodeFormalBuilds.Add(Codeless.Key, NewPlatList);
}
}
else
{
Log(System.Diagnostics.TraceEventType.Information, "{0} was listed as a codeless formal build GUBP_GetNonCodeFormalBuilds_BaseEditorTypeOnly, however it does not exist in this branch.", Codeless.Key);
}
}
}
DoASharedPromotable = NumSharedCode > 0 || NonCodeProjectNames.Count > 0 || NonCodeFormalBuilds.Count > 0;
AddNode(new NonUnityTestNode(HostPlatform));
if (DoASharedPromotable)
{
var AgentSharingGroup = "Shared_EditorTests" + HostPlatformNode.StaticGetHostPlatformSuffix(HostPlatform);
var Options = Branch.BaseEngineProject.Options(HostPlatform);
if (!Options.bIsPromotable || Options.bSeparateGamePromotion)
{
throw new AutomationException("We assume that if we have shared promotable, the base engine is in it.");
}
if (HostPlatform == UnrealTargetPlatform.Win64) //temp hack till automated testing works on other platforms than Win64
{
var EditorTests = Branch.BaseEngineProject.Properties.Targets[TargetRules.TargetType.Editor].Rules.GUBP_GetEditorTests_EditorTypeOnly(HostPlatform);
var EditorTestNodes = new List<string>();
foreach (var Test in EditorTests)
{
if (!bNoAutomatedTesting)
{
EditorTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, Branch.BaseEngineProject, Test.Key, Test.Value, AgentSharingGroup)));
foreach (var NonCodeProject in Branch.NonCodeProjects)
{
if (!NonCodeProjectNames.ContainsKey(NonCodeProject.GameName))
{
continue;
}
if (HostPlatform == UnrealTargetPlatform.Mac) continue; //temp hack till mac automated testing works
EditorTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, NonCodeProject, Test.Key, Test.Value, AgentSharingGroup)));
}
}
}
if (EditorTestNodes.Count > 0)
{
AddNode(new GameAggregateNode(this, HostPlatform, Branch.BaseEngineProject, "AllEditorTests", EditorTestNodes, 0.0f));
}
}
var ServerPlatforms = new List<UnrealTargetPlatform>();
var GamePlatforms = new List<UnrealTargetPlatform>();
foreach (var Kind in BranchInfo.MonolithicKinds)
{
if (Branch.BaseEngineProject.Properties.Targets.ContainsKey(Kind))
{
var Target = Branch.BaseEngineProject.Properties.Targets[Kind];
var Platforms = Target.Rules.GUBP_GetPlatforms_MonolithicOnly(HostPlatform);
if (Platforms.Contains(HostPlatform))
{
// we want the host platform first since some some pseudodependencies look to see if the shared promotable exists.
Platforms.Remove(HostPlatform);
Platforms.Insert(0, HostPlatform);
}
foreach (var Plat in Platforms)
{
if (!Platform.Platforms[HostPlatform].CanHostPlatform(Plat))
{
throw new AutomationException("Project {0} asked for platform {1} with host {2}, but the host platform cannot build that platform.", Branch.BaseEngineProject.GameName, Plat.ToString(), HostPlatform.ToString());
}
if (bNoIOSOnPC && Plat == UnrealTargetPlatform.IOS && HostPlatform == UnrealTargetPlatform.Win64)
{
continue;
}
if (ActivePlatforms.Contains(Plat))
{
if (Kind == TargetRules.TargetType.Server && !ServerPlatforms.Contains(Plat))
{
ServerPlatforms.Add(Plat);
}
if (Kind == TargetRules.TargetType.Game && !GamePlatforms.Contains(Plat))
{
GamePlatforms.Add(Plat);
}
if (!GUBPNodes.ContainsKey(GamePlatformMonolithicsNode.StaticGetFullName(HostPlatform, Branch.BaseEngineProject, Plat)))
{
if(GamePlatformMonolithicsNode.HasPrecompiledTargets(Branch.BaseEngineProject, HostPlatform, Plat))
{
AddNode(new GamePlatformMonolithicsNode(this, HostPlatform, Branch.BaseEngineProject, Plat, InPrecompiled: true));
}
AddNode(new GamePlatformMonolithicsNode(this, HostPlatform, Branch.BaseEngineProject, Plat));
}
if (Plat == UnrealTargetPlatform.Win32 && Target.Rules.GUBP_BuildWindowsXPMonolithics() && Kind == TargetRules.TargetType.Game)
{
if (!GUBPNodes.ContainsKey(GamePlatformMonolithicsNode.StaticGetFullName(HostPlatform, Branch.BaseEngineProject, Plat, true)))
{
AddNode(new GamePlatformMonolithicsNode(this, HostPlatform, Branch.BaseEngineProject, Plat, true));
}
}
}
}
}
}
var CookedAgentSharingGroup = "Shared_CookedTests" + HostPlatformNode.StaticGetHostPlatformSuffix(HostPlatform);
var GameTestNodes = new List<string>();
var GameCookNodes = new List<string>();
//var FormalAgentSharingGroup = "Shared_FormalBuilds" + HostPlatformNode.StaticGetHostPlatformSuffix(HostPlatform);
//foreach (var Kind in BranchInfo.MonolithicKinds)//for now, non-code projects don't do client or server.
{
var Kind = TargetRules.TargetType.Game;
if (Branch.BaseEngineProject.Properties.Targets.ContainsKey(Kind))
{
var Target = Branch.BaseEngineProject.Properties.Targets[Kind];
var Platforms = Target.Rules.GUBP_GetPlatforms_MonolithicOnly(HostPlatform);
foreach (var Plat in Platforms)
{
if (!Platform.Platforms[HostPlatform].CanHostPlatform(Plat))
{
throw new AutomationException("Project {0} asked for platform {1} with host {2}, but the host platform cannot build that platform.", Branch.BaseEngineProject.GameName, Plat.ToString(), HostPlatform.ToString());
}
if (bNoIOSOnPC && Plat == UnrealTargetPlatform.IOS && HostPlatform == UnrealTargetPlatform.Win64)
{
continue;
}
if (ActivePlatforms.Contains(Plat))
{
string CookedPlatform = Platform.Platforms[Plat].GetCookPlatform(Kind == TargetRules.TargetType.Server, Kind == TargetRules.TargetType.Client, "");
if (!GUBPNodes.ContainsKey(CookNode.StaticGetFullName(HostPlatform, Branch.BaseEngineProject, CookedPlatform)))
{
GameCookNodes.Add(AddNode(new CookNode(this, HostPlatform, Branch.BaseEngineProject, Plat, CookedPlatform)));
}
if (!GUBPNodes.ContainsKey(GamePlatformCookedAndCompiledNode.StaticGetFullName(HostPlatform, Branch.BaseEngineProject, Plat)))
{
AddNode(new GamePlatformCookedAndCompiledNode(this, HostPlatform, Branch.BaseEngineProject, Plat, false));
}
var GameTests = Target.Rules.GUBP_GetGameTests_MonolithicOnly(HostPlatform, GetAltHostPlatform(HostPlatform), Plat);
var RequiredPlatforms = new List<UnrealTargetPlatform> { Plat };
if (!bNoAutomatedTesting)
{
var ThisMonoGameTestNodes = new List<string>();
foreach (var Test in GameTests)
{
var TestName = Test.Key + "_" + Plat.ToString();
ThisMonoGameTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, Branch.BaseEngineProject, TestName, Test.Value, CookedAgentSharingGroup, false, RequiredPlatforms)));
}
if (ThisMonoGameTestNodes.Count > 0)
{
GameTestNodes.Add(AddNode(new GameAggregateNode(this, HostPlatform, Branch.BaseEngineProject, "CookedTests_" + Plat.ToString() + "_" + Kind.ToString() + HostPlatformNode.StaticGetHostPlatformSuffix(HostPlatform), ThisMonoGameTestNodes, 0.0f)));
}
}
foreach (var NonCodeProject in Branch.NonCodeProjects)
{
if (!NonCodeProjectNames.ContainsKey(NonCodeProject.GameName) || !NonCodeProjectNames[NonCodeProject.GameName].Contains(Plat))
{
continue;
}
if (!GUBPNodes.ContainsKey(CookNode.StaticGetFullName(HostPlatform, NonCodeProject, CookedPlatform)))
{
GameCookNodes.Add(AddNode(new CookNode(this, HostPlatform, NonCodeProject, Plat, CookedPlatform)));
}
if (!GUBPNodes.ContainsKey(GamePlatformCookedAndCompiledNode.StaticGetFullName(HostPlatform, NonCodeProject, Plat)))
{
AddNode(new GamePlatformCookedAndCompiledNode(this, HostPlatform, NonCodeProject, Plat, false));
if (NonCodeFormalBuilds.ContainsKey(NonCodeProject.GameName))
{
var PlatList = NonCodeFormalBuilds[NonCodeProject.GameName];
foreach (var PlatPair in PlatList)
{
if (PlatPair.TargetPlatform == Plat)
{
var NodeName = AddNode(new FormalBuildNode(this, NonCodeProject, HostPlatform, new List<UnrealTargetPlatform>() { Plat }, new List<UnrealTargetConfiguration>() {PlatPair.TargetConfig}));
if(PlatPair.bBeforeTrigger)
{
RemovePseudodependencyFromNode(FormalBuildNode.StaticGetFullName(NonCodeProject, HostPlatform, new List<UnrealTargetPlatform>() { Plat }, new List<UnrealTargetConfiguration>() { PlatPair.TargetConfig }), WaitForFormalUserInput.StaticGetFullName());
}
// we don't want this delayed
// this would normally wait for the testing phase, we just want to build it right away
RemovePseudodependencyFromNode(
CookNode.StaticGetFullName(HostPlatform, NonCodeProject, CookedPlatform),
WaitForTestShared.StaticGetFullName());
string BuildAgentSharingGroup = "";
if (Options.bSeparateGamePromotion)
{
BuildAgentSharingGroup = NonCodeProject.GameName + "_MakeFormalBuild_" + Plat.ToString() + HostPlatformNode.StaticGetHostPlatformSuffix(HostPlatform);
if (Plat == UnrealTargetPlatform.IOS || Plat == UnrealTargetPlatform.Android) // These trash build products, so we need to use different agents
{
BuildAgentSharingGroup = "";
}
GUBPNodes[CookNode.StaticGetFullName(HostPlatform, NonCodeProject, CookedPlatform)].AgentSharingGroup = BuildAgentSharingGroup;
GUBPNodes[GamePlatformCookedAndCompiledNode.StaticGetFullName(HostPlatform, NonCodeProject, Plat)].AgentSharingGroup = BuildAgentSharingGroup;
GUBPNodes[NodeName].AgentSharingGroup = BuildAgentSharingGroup;
}
else
{
//GUBPNodes[NodeName].AgentSharingGroup = FormalAgentSharingGroup;
if(Plat == UnrealTargetPlatform.XboxOne)
{
GUBPNodes[NodeName].AgentSharingGroup = "";
}
}
if (PlatPair.bTest)
{
AddNode(new FormalBuildTestNode(this, NonCodeProject, HostPlatform, Plat, PlatPair.TargetConfig));
}
}
}
}
}
if (!bNoAutomatedTesting)
{
if (HostPlatform == UnrealTargetPlatform.Mac || HostPlatform == UnrealTargetPlatform.Linux) continue; //temp hack till Linux and Mac automated testing works
var ThisMonoGameTestNodes = new List<string>();
foreach (var Test in GameTests)
{
var TestName = Test.Key + "_" + Plat.ToString();
ThisMonoGameTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, NonCodeProject, TestName, Test.Value, CookedAgentSharingGroup, false, RequiredPlatforms)));
}
if (ThisMonoGameTestNodes.Count > 0)
{
GameTestNodes.Add(AddNode(new GameAggregateNode(this, HostPlatform, NonCodeProject, "CookedTests_" + Plat.ToString() + "_" + Kind.ToString() + HostPlatformNode.StaticGetHostPlatformSuffix(HostPlatform), ThisMonoGameTestNodes, 0.0f)));
}
}
}
}
}
}
}
#if false
//for now, non-code projects don't do client or server.
foreach (var ServerPlatform in ServerPlatforms)
{
var ServerTarget = Branch.BaseEngineProject.Properties.Targets[TargetRules.TargetType.Server];
foreach (var GamePlatform in GamePlatforms)
{
var Target = Branch.BaseEngineProject.Properties.Targets[TargetRules.TargetType.Game];
foreach (var NonCodeProject in Branch.NonCodeProjects)
{
if (!NonCodeProjectNames.ContainsKey(NonCodeProject.GameName) || !NonCodeProjectNames.ContainsKey(NonCodeProject.GameName) ||
!NonCodeProjectNames[NonCodeProject.GameName].Contains(ServerPlatform) || !NonCodeProjectNames[NonCodeProject.GameName].Contains(GamePlatform) )
{
continue;
}
var ClientServerTests = Target.Rules.GUBP_GetClientServerTests_MonolithicOnly(HostPlatform, GetAltHostPlatform(HostPlatform), ServerPlatform, GamePlatform);
var RequiredPlatforms = new List<UnrealTargetPlatform> { ServerPlatform };
if (ServerPlatform != GamePlatform)
{
RequiredPlatforms.Add(GamePlatform);
}
foreach (var Test in ClientServerTests)
{
GameTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, NonCodeProject, Test.Key + "_" + GamePlatform.ToString() + "_" + ServerPlatform.ToString(), Test.Value, false, RequiredPlatforms, true)));
}
}
}
}
#endif
if (GameTestNodes.Count > 0)
{
AddNode(new GameAggregateNode(this, HostPlatform, Branch.BaseEngineProject, "AllCookedTests", GameTestNodes));
}
}
if (!GUBPNodes.ContainsKey(SharedCookAggregateNode.StaticGetFullName()))
{
AddNode(new SharedCookAggregateNode(this, HostPlatforms, NonCodeProjectNames, NonCodeFormalBuilds));
}
if(HostPlatform == MakeFeaturePacksNode.GetDefaultBuildPlatform(this))
{
AddNode(new MakeFeaturePacksNode(HostPlatform, Branch.AllProjects.Where(x => MakeFeaturePacksNode.IsFeaturePack(x))));
}
foreach (var CodeProj in Branch.CodeProjects)
{
var Options = CodeProj.Options(HostPlatform);
if (!Options.bIsPromotable && !Options.bTestWithShared && !Options.bIsNonCode)
{
continue; // we skip things that aren't promotable and aren't tested - except noncode as code situations
}
var AgentShareName = CodeProj.GameName;
if (!Options.bSeparateGamePromotion)
{
AgentShareName = "Shared";
}
if (!BranchOptions.ExcludePlatformsForEditor.Contains(HostPlatform) && !Options.bIsNonCode)
{
EditorGameNode Node = (EditorGameNode)TryFindNode(EditorGameNode.StaticGetFullName(HostPlatform, CodeProj));
if(Node == null)
{
AddNode(new EditorGameNode(this, HostPlatform, CodeProj));
}
else
{
Node.AddProject(CodeProj);
}
}
if (!bNoAutomatedTesting && HostPlatform == UnrealTargetPlatform.Win64) //temp hack till automated testing works on other platforms than Win64
{
if (CodeProj.Properties.Targets.ContainsKey(TargetRules.TargetType.Editor))
{
var EditorTests = CodeProj.Properties.Targets[TargetRules.TargetType.Editor].Rules.GUBP_GetEditorTests_EditorTypeOnly(HostPlatform);
var EditorTestNodes = new List<string>();
string AgentSharingGroup = "";
if (EditorTests.Count > 1)
{
AgentSharingGroup = AgentShareName + "_EditorTests" + HostPlatformNode.StaticGetHostPlatformSuffix(HostPlatform);
}
foreach (var Test in EditorTests)
{
EditorTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, CodeProj, Test.Key, Test.Value, AgentSharingGroup)));
if (!Options.bTestWithShared || !HasNode(WaitForTestShared.StaticGetFullName()))
{
RemovePseudodependencyFromNode((UATTestNode.StaticGetFullName(HostPlatform, CodeProj, Test.Key)), WaitForTestShared.StaticGetFullName());
}
}
if (EditorTestNodes.Count > 0)
{
AddNode(new GameAggregateNode(this, HostPlatform, CodeProj, "AllEditorTests", EditorTestNodes, 0.0f));
}
}
}
var CookedAgentSharingGroup = AgentShareName + "_CookedTests" + HostPlatformNode.StaticGetHostPlatformSuffix(HostPlatform);
//var FormalAgentSharingGroup = "Shared_FormalBuilds" + HostPlatformNode.StaticGetHostPlatformSuffix(HostPlatform);
var ServerPlatforms = new List<UnrealTargetPlatform>();
var GamePlatforms = new List<UnrealTargetPlatform>();
var GameTestNodes = new List<string>();
foreach (var Kind in BranchInfo.MonolithicKinds)
{
if (CodeProj.Properties.Targets.ContainsKey(Kind))
{
var Target = CodeProj.Properties.Targets[Kind];
var Platforms = Target.Rules.GUBP_GetPlatforms_MonolithicOnly(HostPlatform);
var AdditionalPlatforms = Target.Rules.GUBP_GetBuildOnlyPlatforms_MonolithicOnly(HostPlatform);
var AllPlatforms = Platforms.Union(AdditionalPlatforms);
foreach (var Plat in AllPlatforms)
{
if (!Platform.Platforms[HostPlatform].CanHostPlatform(Plat))
{
throw new AutomationException("Project {0} asked for platform {1} with host {2}, but the host platform cannot build that platform.", CodeProj.GameName, Plat.ToString(), HostPlatform.ToString());
}
if (bNoIOSOnPC && Plat == UnrealTargetPlatform.IOS && HostPlatform == UnrealTargetPlatform.Win64)
{
continue;
}
if(Plat == UnrealTargetPlatform.Win32 && Target.Rules.GUBP_BuildWindowsXPMonolithics())
{
if(!GUBPNodes.ContainsKey(GamePlatformMonolithicsNode.StaticGetFullName(HostPlatform, CodeProj, Plat, true)))
{
AddNode(new GamePlatformMonolithicsNode(this, HostPlatform, CodeProj, Plat, true));
}
}
if (ActivePlatforms.Contains(Plat))
{
if (Kind == TargetRules.TargetType.Server && !ServerPlatforms.Contains(Plat))
{
ServerPlatforms.Add(Plat);
}
if (Kind == TargetRules.TargetType.Game && !GamePlatforms.Contains(Plat))
{
GamePlatforms.Add(Plat);
}
if (!GUBPNodes.ContainsKey(GamePlatformMonolithicsNode.StaticGetFullName(HostPlatform, CodeProj, Plat)))
{
if(GamePlatformMonolithicsNode.HasPrecompiledTargets(CodeProj, HostPlatform, Plat))
{
AddNode(new GamePlatformMonolithicsNode(this, HostPlatform, CodeProj, Plat, InPrecompiled: true));
}
AddNode(new GamePlatformMonolithicsNode(this, HostPlatform, CodeProj, Plat));
}
if (!AdditionalPlatforms.Contains(Plat))
{
string CookedPlatform = Platform.Platforms[Plat].GetCookPlatform(Kind == TargetRules.TargetType.Server, Kind == TargetRules.TargetType.Client, "");
if (Target.Rules.GUBP_AlternateCookPlatform(HostPlatform, CookedPlatform) != "")
{
CookedPlatform = Target.Rules.GUBP_AlternateCookPlatform(HostPlatform, CookedPlatform);
}
if (!GUBPNodes.ContainsKey(CookNode.StaticGetFullName(HostPlatform, CodeProj, CookedPlatform)))
{
AddNode(new CookNode(this, HostPlatform, CodeProj, Plat, CookedPlatform));
}
if (!GUBPNodes.ContainsKey(GamePlatformCookedAndCompiledNode.StaticGetFullName(HostPlatform, CodeProj, Plat)))
{
AddNode(new GamePlatformCookedAndCompiledNode(this, HostPlatform, CodeProj, Plat, true));
}
var FormalBuildConfigs = Target.Rules.GUBP_GetConfigsForFormalBuilds_MonolithicOnly(HostPlatform);
foreach (var Config in FormalBuildConfigs)
{
string FormalNodeName = null;
if (Kind == TargetRules.TargetType.Client)
{
if (Plat == Config.TargetPlatform)
{
FormalNodeName = AddNode(new FormalBuildNode(this, CodeProj, HostPlatform, InClientTargetPlatforms: new List<UnrealTargetPlatform>() { Config.TargetPlatform }, InClientConfigs: new List<UnrealTargetConfiguration>() { Config.TargetConfig }, InClientNotGame: true));
}
}
else if (Kind == TargetRules.TargetType.Server)
{
if (Plat == Config.TargetPlatform)
{
FormalNodeName = AddNode(new FormalBuildNode(this, CodeProj, HostPlatform, InServerTargetPlatforms: new List<UnrealTargetPlatform>() { Config.TargetPlatform }, InServerConfigs: new List<UnrealTargetConfiguration>() { Config.TargetConfig }));
}
}
else if (Kind == TargetRules.TargetType.Game)
{
if (Plat == Config.TargetPlatform)
{
FormalNodeName = AddNode(new FormalBuildNode(this, CodeProj, HostPlatform, InClientTargetPlatforms: new List<UnrealTargetPlatform>() { Config.TargetPlatform }, InClientConfigs: new List<UnrealTargetConfiguration>() { Config.TargetConfig }));
}
}
if (FormalNodeName != null)
{
if (Config.bBeforeTrigger)
{
RemovePseudodependencyFromNode(FormalBuildNode.StaticGetFullName(CodeProj, HostPlatform, new List<UnrealTargetPlatform>() { Config.TargetPlatform }, new List<UnrealTargetConfiguration>() { Config.TargetConfig }), WaitForFormalUserInput.StaticGetFullName());
}
// we don't want this delayed
// this would normally wait for the testing phase, we just want to build it right away
RemovePseudodependencyFromNode(
CookNode.StaticGetFullName(HostPlatform, CodeProj, CookedPlatform),
WaitForTestShared.StaticGetFullName());
string BuildAgentSharingGroup = "";
if (Options.bSeparateGamePromotion)
{
BuildAgentSharingGroup = CodeProj.GameName + "_MakeFormalBuild_" + Plat.ToString() + HostPlatformNode.StaticGetHostPlatformSuffix(HostPlatform);
if (Plat == UnrealTargetPlatform.IOS || Plat == UnrealTargetPlatform.Android || Plat == UnrealTargetPlatform.XboxOne) // These trash build products, so we need to use different agents
{
BuildAgentSharingGroup = "";
}
GUBPNodes[CookNode.StaticGetFullName(HostPlatform, CodeProj, CookedPlatform)].AgentSharingGroup = BuildAgentSharingGroup;
GUBPNodes[GamePlatformCookedAndCompiledNode.StaticGetFullName(HostPlatform, CodeProj, Plat)].AgentSharingGroup = BuildAgentSharingGroup;
GUBPNodes[FormalNodeName].AgentSharingGroup = BuildAgentSharingGroup;
}
else
{
//GUBPNodes[FormalNodeName].AgentSharingGroup = FormalAgentSharingGroup;
if (Plat == UnrealTargetPlatform.XboxOne)
{
GUBPNodes[FormalNodeName].AgentSharingGroup = "";
}
}
if (Config.bTest)
{
AddNode(new FormalBuildTestNode(this, CodeProj, HostPlatform, Plat, Config.TargetConfig));
}
}
}
if (!bNoAutomatedTesting)
{
if (HostPlatform == UnrealTargetPlatform.Mac || HostPlatform == UnrealTargetPlatform.Linux) continue; //temp hack till Linux and Mac automated testing works
var GameTests = Target.Rules.GUBP_GetGameTests_MonolithicOnly(HostPlatform, GetAltHostPlatform(HostPlatform), Plat);
var RequiredPlatforms = new List<UnrealTargetPlatform> { Plat };
var ThisMonoGameTestNodes = new List<string>();
foreach (var Test in GameTests)
{
var TestNodeName = Test.Key + "_" + Plat.ToString();
ThisMonoGameTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, CodeProj, TestNodeName, Test.Value, CookedAgentSharingGroup, false, RequiredPlatforms)));
if (!Options.bTestWithShared || !HasNode(WaitForTestShared.StaticGetFullName()))
{
RemovePseudodependencyFromNode((UATTestNode.StaticGetFullName(HostPlatform, CodeProj, TestNodeName)), WaitForTestShared.StaticGetFullName());
}
}
if (ThisMonoGameTestNodes.Count > 0)
{
GameTestNodes.Add(AddNode(new GameAggregateNode(this, HostPlatform, CodeProj, "CookedTests_" + Plat.ToString() + "_" + Kind.ToString() + HostPlatformNode.StaticGetHostPlatformSuffix(HostPlatform), ThisMonoGameTestNodes, 0.0f)));
}
}
}
}
}
}
}
if (!bNoAutomatedTesting)
{
foreach (var ServerPlatform in ServerPlatforms)
{
foreach (var GamePlatform in GamePlatforms)
{
if (HostPlatform == UnrealTargetPlatform.Mac || HostPlatform == UnrealTargetPlatform.Linux) continue; //temp hack till Linux and Mac automated testing works
var Target = CodeProj.Properties.Targets[TargetRules.TargetType.Game];
var ClientServerTests = Target.Rules.GUBP_GetClientServerTests_MonolithicOnly(HostPlatform, GetAltHostPlatform(HostPlatform), ServerPlatform, GamePlatform);
var RequiredPlatforms = new List<UnrealTargetPlatform> { ServerPlatform };
if (ServerPlatform != GamePlatform)
{
RequiredPlatforms.Add(GamePlatform);
}
foreach (var Test in ClientServerTests)
{
var TestNodeName = Test.Key + "_" + GamePlatform.ToString() + "_" + ServerPlatform.ToString();
GameTestNodes.Add(AddNode(new UATTestNode(this, HostPlatform, CodeProj, TestNodeName, Test.Value, CookedAgentSharingGroup, false, RequiredPlatforms)));
}
}
}
if (GameTestNodes.Count > 0)
{
AddNode(new GameAggregateNode(this, HostPlatform, CodeProj, "AllCookedTests", GameTestNodes));
}
}
}
}
int NumSharedAllHosts = 0;
foreach (var CodeProj in Branch.CodeProjects)
{
if (CodeProj.Properties.Targets.ContainsKey(TargetRules.TargetType.Editor))
{
bool AnySeparate = false;
var PromotedHosts = new List<UnrealTargetPlatform>();
foreach (var HostPlatform in HostPlatforms)
{
if (!BranchOptions.ExcludePlatformsForEditor.Contains(HostPlatform) && !BranchOptions.RemovePlatformFromPromotable.Contains(HostPlatform))
{
var Options = CodeProj.Options(HostPlatform);
AnySeparate = AnySeparate || Options.bSeparateGamePromotion;
if (Options.bIsPromotable)
{
if (!Options.bSeparateGamePromotion)
{
NumSharedAllHosts++;
}
PromotedHosts.Add(HostPlatform);
}
}
}
if (PromotedHosts.Count > 0)
{
AddNode(new GameAggregatePromotableNode(this, PromotedHosts, CodeProj, true));
if (AnySeparate)
{
AddNode(new WaitForGamePromotionUserInput(this, CodeProj, false));
AddNode(new GameLabelPromotableNode(this, CodeProj, false));
AddNode(new WaitForGamePromotionUserInput(this, CodeProj, true));
AddNode(new GameLabelPromotableNode(this, CodeProj, true));
}
}
}
}
if (NumSharedAllHosts > 0)
{
AddNode(new GameAggregatePromotableNode(this, HostPlatforms, Branch.BaseEngineProject, false));
AddNode(new SharedAggregatePromotableNode(this, HostPlatforms));
AddNode(new WaitForSharedPromotionUserInput(this, false));
AddNode(new SharedLabelPromotableNode(this, false));
AddNode(new SharedLabelPromotableSuccessNode());
AddNode(new WaitForTestShared(this));
AddNode(new WaitForSharedPromotionUserInput(this, true));
AddNode(new SharedLabelPromotableNode(this, true));
AddNode(new WaitForFormalUserInput(this));
}
foreach (var HostPlatform in HostPlatforms)
{
AddCustomNodes(HostPlatform);
}
if (HasNode(ToolsForCompileNode.StaticGetFullName(UnrealTargetPlatform.Win64)))
{
if (HasNode(GamePlatformMonolithicsNode.StaticGetFullName(UnrealTargetPlatform.Mac, Branch.BaseEngineProject, UnrealTargetPlatform.IOS)) && HasNode(ToolsNode.StaticGetFullName(UnrealTargetPlatform.Win64)))
{
//AddNode(new IOSOnPCTestNode(this)); - Disable IOSOnPCTest until a1011 crash is fixed
}
AddNode(new VSExpressTestNode(this));
if (ActivePlatforms.Contains(UnrealTargetPlatform.Linux) && !BranchOptions.ExcludePlatformsForEditor.Contains(UnrealTargetPlatform.Linux))
{
AddNode(new RootEditorCrossCompileLinuxNode(UnrealTargetPlatform.Win64));
}
if (!bPreflightBuild)
{
AddNode(new CleanSharedTempStorageNode(this));
}
}
#if false
// this doesn't work for lots of reasons...we can't figure out what the dependencies are until far later
if (bPreflightBuild)
{
GeneralSuccessNode PreflightSuccessNode = new GeneralSuccessNode("Preflight");
foreach (var NodeToDo in GUBPNodes)
{
if (NodeToDo.Value.RunInEC())
{
PreflightSuccessNode.AddPseudodependency(NodeToDo.Key);
}
}
AddNode(PreflightSuccessNode);
}
#endif
foreach (var NodeToDo in GUBPNodes)
{
foreach (var Dep in GUBPNodes[NodeToDo.Key].FullNamesOfDependencies)
{
if (!GUBPNodes.ContainsKey(Dep))
{
throw new AutomationException("Node {0} is not in the full graph. It is a dependency of {1}.", Dep, NodeToDo.Key);
}
if (Dep == NodeToDo.Key)
{
throw new AutomationException("Node {0} has a self arc.", NodeToDo.Key);
}
}
foreach (var Dep in GUBPNodes[NodeToDo.Key].FullNamesOfPseudosependencies)
{
if (!GUBPNodes.ContainsKey(Dep))
{
throw new AutomationException("Node {0} is not in the full graph. It is a pseudodependency of {1}.", Dep, NodeToDo.Key);
}
if (Dep == NodeToDo.Key)
{
throw new AutomationException("Node {0} has a self pseudoarc.", NodeToDo.Key);
}
}
}
if (bCleanLocalTempStorage) // shared temp storage can never be wiped
{
DeleteLocalTempStorageManifests(CmdEnv);
}
GUBPNodesControllingTrigger = new Dictionary<string, string>();
GUBPNodesControllingTriggerDotName = new Dictionary<string, string>();
var FullNodeList = new Dictionary<string, string>();
var FullNodeDirectDependencies = new Dictionary<string, string>();
var FullNodeDependedOnBy = new Dictionary<string, string>();
var FullNodeDependentPromotions = new Dictionary<string, string>();
var SeparatePromotables = new List<string>();
{
foreach (var NodeToDo in GUBPNodes)
{
if (GUBPNodes[NodeToDo.Key].IsSeparatePromotable())
{
SeparatePromotables.Add(GUBPNodes[NodeToDo.Key].GetFullName());
List<string> Dependencies = new List<string>();
Dependencies = GetECDependencies(NodeToDo.Key);
foreach (var Dep in Dependencies)
{
if (!GUBPNodes.ContainsKey(Dep))
{
throw new AutomationException("Node {0} is not in the graph. It is a dependency of {1}.", Dep, NodeToDo);
}
if (!GUBPNodes[Dep].IsPromotableAggregate())
{
if (!GUBPNodes[Dep].DependentPromotions.Contains(NodeToDo.Key))
{
GUBPNodes[Dep].DependentPromotions.Add(NodeToDo.Key);
}
}
}
}
}
// Make sure that everything that's listed as a frequency barrier is completed with the given interval
Dictionary<string, int> FrequencyOverrides = new Dictionary<string,int>();
foreach (KeyValuePair<string, sbyte> Barrier in BranchOptions.FrequencyBarriers)
{
// All the nodes which are dependencies of the barrier node
HashSet<string> IncludedNodes = new HashSet<string> { Barrier.Key };
// Find all the nodes which are indirect dependencies of this node
List<string> SearchNodes = new List<string> { Barrier.Key };
for (int Idx = 0; Idx < SearchNodes.Count; Idx++)
{
GUBPNode Node = GUBPNodes[SearchNodes[Idx]];
foreach (string DependencyName in Node.FullNamesOfDependencies.Union(Node.FullNamesOfPseudosependencies))
{
if (!IncludedNodes.Contains(DependencyName))
{
IncludedNodes.Add(DependencyName);
SearchNodes.Add(DependencyName);
}
}
}
// Make sure that everything included in this list is before the cap, and everything not in the list is after it
foreach (KeyValuePair<string, GUBPNode> NodePair in GUBPNodes)
{
if (IncludedNodes.Contains(NodePair.Key))
{
int Frequency;
if(FrequencyOverrides.TryGetValue(NodePair.Key, out Frequency))
{
Frequency = Math.Min(Frequency, Barrier.Value);
}
else
{
Frequency = Barrier.Value;
}
FrequencyOverrides[NodePair.Key] = Frequency;
}
}
}
// Compute all the frequencies
foreach (var NodeToDo in GUBPNodes)
{
ComputeDependentCISFrequencyQuantumShift(NodeToDo.Key, FrequencyOverrides);
}
foreach (var NodeToDo in GUBPNodes)
{
var Deps = GUBPNodes[NodeToDo.Key].DependentPromotions;
string All = "";
foreach (var Dep in Deps)
{
if (All != "")
{
All += " ";
}
All += Dep;
}
FullNodeDependentPromotions.Add(NodeToDo.Key, All);
}
}
{
Log("******* {0} GUBP Nodes", GUBPNodes.Count);
var SortedNodes = TopologicalSort(new HashSet<string>(GUBPNodes.Keys), LocalOnly: true, DoNotConsiderCompletion: true);
foreach (var Node in SortedNodes)
{
string Note = GetControllingTriggerDotName(Node);
if (Note == "")
{
Note = CISFrequencyQuantumShiftString(Node);
}
if (Note == "")
{
Note = "always";
}
if (GUBPNodes[Node].RunInEC())
{
var Deps = GetECDependencies(Node);
string All = "";
foreach (var Dep in Deps)
{
if (All != "")
{
All += " ";
}
All += Dep;
}
Log(" {0}: {1} {2}", Node, Note, All);
FullNodeList.Add(Node, Note);
FullNodeDirectDependencies.Add(Node, All);
}
else
{
Log(" {0}: {1} [Aggregate]", Node, Note);
}
}
}
Dictionary<string, int> FullNodeListSortKey = GetDisplayOrder(FullNodeList.Keys.ToList(), FullNodeDirectDependencies, GUBPNodes);
bool bOnlyNode = false;
bool bRelatedToNode = false;
bool bGraphSubset = false;
var NodesToDo = new HashSet<string>();
{
string NodeSpec = ParseParamValue("Node");
if (String.IsNullOrEmpty(NodeSpec))
{
NodeSpec = ParseParamValue("RelatedToNode");
if (!String.IsNullOrEmpty(NodeSpec))
{
bRelatedToNode = true;
}
}
if (String.IsNullOrEmpty(NodeSpec) && CommanderSetup)
{
NodeSpec = ParseParamValue("SetupNode");
if (String.IsNullOrEmpty(NodeSpec))
{
NodeSpec = ParseParamValue("SetupRelatedToNode");
if (!String.IsNullOrEmpty(NodeSpec))
{
bRelatedToNode = true;
}
}
}
if (String.IsNullOrEmpty(NodeSpec))
{
NodeSpec = ParseParamValue("OnlyNode");
if (!String.IsNullOrEmpty(NodeSpec))
{
bOnlyNode = true;
}
}
if (!String.IsNullOrEmpty(NodeSpec))
{
bGraphSubset = true;
if (NodeSpec.Equals("Noop", StringComparison.InvariantCultureIgnoreCase))
{
Log("Request for Noop node, done.");
PrintRunTime();
return;
}
List<string> Nodes = new List<string>(NodeSpec.Split('+'));
foreach (var NodeArg in Nodes)
{
var NodeName = NodeArg.Trim();
bool bFoundAnything = false;
if (!String.IsNullOrEmpty(NodeName))
{
foreach (var Node in GUBPNodes)
{
if (Node.Value.GetFullName().Equals(NodeArg, StringComparison.InvariantCultureIgnoreCase) ||
Node.Value.AgentSharingGroup.Equals(NodeArg, StringComparison.InvariantCultureIgnoreCase)
)
{
if (!NodesToDo.Contains(Node.Key))
{
NodesToDo.Add(Node.Key);
}
bFoundAnything = true;
}
}
if (!bFoundAnything)
{
throw new AutomationException("Could not find node named {0}", NodeName);
}
}
}
}
}
string GameSpec = ParseParamValue("Game");
if (!String.IsNullOrEmpty(GameSpec))
{
bGraphSubset = true;
List<string> Games = new List<string>(GameSpec.Split('+'));
foreach (var GameArg in Games)
{
var GameName = GameArg.Trim();
if (!String.IsNullOrEmpty(GameName))
{
foreach (var GameProj in Branch.CodeProjects)
{
if (GameProj.GameName.Equals(GameName, StringComparison.InvariantCultureIgnoreCase))
{
NodesToDo.Add(GameAggregatePromotableNode.StaticGetFullName(GameProj));
foreach (var Node in GUBPNodes)
{
if (Node.Value.GameNameIfAnyForTempStorage() == GameProj.GameName)
{
NodesToDo.Add(Node.Key);
}
}
GameName = null;
}
}
if (GameName != null)
{
foreach (var GameProj in Branch.NonCodeProjects)
{
if (GameProj.GameName.Equals(GameName, StringComparison.InvariantCultureIgnoreCase))
{
foreach (var Node in GUBPNodes)
{
if (Node.Value.GameNameIfAnyForTempStorage() == GameProj.GameName)
{
NodesToDo.Add(Node.Key);
}
}
GameName = null;
}
}
}
if (GameName != null)
{
throw new AutomationException("Could not find game named {0}", GameName);
}
}
}
}
if (NodesToDo.Count == 0)
{
Log("No nodes specified, adding all nodes");
foreach (var Node in GUBPNodes)
{
NodesToDo.Add(Node.Key);
}
}
else if (TimeIndex != 0)
{
Log("Check to make sure we didn't ask for nodes that will be culled by time index");
foreach (var NodeToDo in NodesToDo)
{
if (TimeIndex % (1 << GUBPNodes[NodeToDo].DependentCISFrequencyQuantumShift()) != 0)
{
throw new AutomationException("You asked specifically for node {0}, but it is culled by the time quantum: TimeIndex = {1}, DependentCISFrequencyQuantumShift = {2}.", NodeToDo, TimeIndex, GUBPNodes[NodeToDo].DependentCISFrequencyQuantumShift());
}
}
}
Log("Desired Nodes");
foreach (var NodeToDo in NodesToDo)
{
Log(" {0}", NodeToDo);
}
// if we are doing related to, then find things that depend on the selected nodes
if (bRelatedToNode)
{
bool bDoneWithDependencies = false;
while (!bDoneWithDependencies)
{
bDoneWithDependencies = true;
var Fringe = new HashSet<string>();
foreach (var NodeToDo in GUBPNodes)
{
if (!NodesToDo.Contains(NodeToDo.Key))
{
foreach (var Dep in GUBPNodes[NodeToDo.Key].FullNamesOfDependencies)
{
if (!GUBPNodes.ContainsKey(Dep))
{
throw new AutomationException("Node {0} is not in the graph. It is a dependency of {1}.", Dep, NodeToDo.Key);
}
if (NodesToDo.Contains(Dep))
{
Fringe.Add(NodeToDo.Key);
bDoneWithDependencies = false;
}
}
foreach (var Dep in GUBPNodes[NodeToDo.Key].FullNamesOfPseudosependencies)
{
if (!GUBPNodes.ContainsKey(Dep))
{
throw new AutomationException("Node {0} is not in the graph. It is a pseudodependency of {1}.", Dep, NodeToDo.Key);
}
}
}
}
NodesToDo.UnionWith(Fringe);
}
}
// find things that our nodes depend on
if (!bOnlyNode)
{
bool bDoneWithDependencies = false;
while (!bDoneWithDependencies)
{
bDoneWithDependencies = true;
var Fringe = new HashSet<string>();
foreach (var NodeToDo in NodesToDo)
{
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfDependencies)
{
if (!GUBPNodes.ContainsKey(Dep))
{
throw new AutomationException("Node {0} is not in the graph. It is a dependency of {1}.", Dep, NodeToDo);
}
if (!NodesToDo.Contains(Dep))
{
Fringe.Add(Dep);
bDoneWithDependencies = false;
}
}
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfPseudosependencies)
{
if (!GUBPNodes.ContainsKey(Dep))
{
throw new AutomationException("Node {0} is not in the graph. It is a pseudodependency of {1}.", Dep, NodeToDo);
}
}
}
NodesToDo.UnionWith(Fringe);
}
}
if (TimeIndex != 0)
{
Log("Culling based on time index");
var NewNodesToDo = new HashSet<string>();
foreach (var NodeToDo in NodesToDo)
{
if (TimeIndex % (1 << GUBPNodes[NodeToDo].DependentCISFrequencyQuantumShift()) == 0)
{
Log(" Keeping {0}", NodeToDo);
NewNodesToDo.Add(NodeToDo);
}
else
{
Log(" Rejecting {0}", NodeToDo);
}
}
NodesToDo = NewNodesToDo;
}
//Remove Plat if specified
if(WithoutLinux)
{
var NewNodesToDo = new HashSet<string>();
foreach(var NodeToDo in NodesToDo)
{
if(!GUBPNodes[NodeToDo].GetFullName().Contains("Linux"))
{
NewNodesToDo.Add(NodeToDo);
}
else
{
Log(" Rejecting {0} because -NoLinux was requested", NodeToDo);
}
}
NodesToDo = NewNodesToDo;
}
//find things that depend on our nodes and setup commander dictionary
if (!bOnlyNode)
{
foreach(var NodeToDo in NodesToDo)
{
if (!GUBPNodes[NodeToDo].IsAggregate() && !GUBPNodes[NodeToDo].IsTest())
{
List<string> ECDependencies = new List<string>();
ECDependencies = GetECDependencies(NodeToDo);
foreach (var Dep in ECDependencies)
{
if (!GUBPNodes.ContainsKey(Dep))
{
throw new AutomationException("Node {0} is not in the graph. It is a dependency of {1}.", Dep, NodeToDo);
}
if (!GUBPNodes[Dep].FullNamesOfDependedOn.Contains(NodeToDo))
{
GUBPNodes[Dep].FullNamesOfDependedOn.Add(NodeToDo);
}
}
}
}
foreach(var NodeToDo in NodesToDo)
{
var Deps = GUBPNodes[NodeToDo].FullNamesOfDependedOn;
string All = "";
foreach (var Dep in Deps)
{
if (All != "")
{
All += " ";
}
All += Dep;
}
FullNodeDependedOnBy.Add(NodeToDo, All);
}
}
if (CommanderSetup)
{
if (!String.IsNullOrEmpty(ExplicitTrigger))
{
bool bFoundIt = false;
foreach (var Node in GUBPNodes)
{
if (Node.Value.GetFullName().Equals(ExplicitTrigger, StringComparison.InvariantCultureIgnoreCase))
{
if (Node.Value.TriggerNode() && Node.Value.RunInEC())
{
Node.Value.SetAsExplicitTrigger();
bFoundIt = true;
break;
}
}
}
if (!bFoundIt)
{
throw new AutomationException("Could not find trigger node named {0}", ExplicitTrigger);
}
}
else
{
if (bSkipTriggers)
{
foreach (var Node in GUBPNodes)
{
if (Node.Value.TriggerNode() && Node.Value.RunInEC())
{
Node.Value.SetAsExplicitTrigger();
}
}
}
}
}
if (bPreflightBuild)
{
Log("Culling triggers and downstream for preflight builds ");
var NewNodesToDo = new HashSet<string>();
foreach (var NodeToDo in NodesToDo)
{
var TriggerDot = GetControllingTriggerDotName(NodeToDo);
if (TriggerDot == "" && !GUBPNodes[NodeToDo].TriggerNode())
{
Log(" Keeping {0}", NodeToDo);
NewNodesToDo.Add(NodeToDo);
}
else
{
Log(" Rejecting {0}", NodeToDo);
}
}
NodesToDo = NewNodesToDo;
}
GUBPNodesCompleted = new Dictionary<string, bool>();
GUBPNodesHistory = new Dictionary<string, NodeHistory>();
Log("******* Caching completion");
{
var StartTime = DateTime.UtcNow;
foreach (var Node in NodesToDo)
{
Log("** {0}", Node);
NodeIsAlreadyComplete(Node, LocalOnly); // cache these now to avoid spam later
GetControllingTriggerDotName(Node);
}
var BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds;
Log("Took {0}s to cache completion for {1} nodes", BuildDuration / 1000, NodesToDo.Count);
}
/*if (CLString != "" && StoreName.Contains(CLString) && !ParseParam("NoHistory"))
{
Log("******* Updating history");
var StartTime = DateTime.UtcNow;
foreach (var Node in NodesToDo)
{
if (!NodeIsAlreadyComplete(Node, LocalOnly))
{
UpdateNodeHistory(Node, CLString);
}
}
var BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds;
Log("Took {0}s to get history for {1} nodes", BuildDuration / 1000, NodesToDo.Count);
}*/
var OrdereredToDo = TopologicalSort(NodesToDo, ExplicitTrigger, LocalOnly);
// find all unfinished triggers, excepting the one we are triggering right now
var UnfinishedTriggers = new List<string>();
if (!bSkipTriggers)
{
foreach (var NodeToDo in OrdereredToDo)
{
if (GUBPNodes[NodeToDo].TriggerNode() && !NodeIsAlreadyComplete(NodeToDo, LocalOnly))
{
if (String.IsNullOrEmpty(ExplicitTrigger) || ExplicitTrigger != NodeToDo)
{
UnfinishedTriggers.Add(NodeToDo);
}
}
}
}
Log("*********** Desired And Dependent Nodes, in order.");
PrintNodes(this, OrdereredToDo, LocalOnly, UnfinishedTriggers);
//check sorting
{
foreach (var NodeToDo in OrdereredToDo)
{
if (GUBPNodes[NodeToDo].TriggerNode() && (GUBPNodes[NodeToDo].IsSticky() || NodeIsAlreadyComplete(NodeToDo, LocalOnly))) // these sticky triggers are ok, everything is already completed anyway
{
continue;
}
if(GUBPNodes[NodeToDo].IsTest())
{
bHasTests = true;
}
int MyIndex = OrdereredToDo.IndexOf(NodeToDo);
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfDependencies)
{
int DepIndex = OrdereredToDo.IndexOf(Dep);
if (DepIndex >= MyIndex)
{
throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo, Dep);
}
}
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfPseudosependencies)
{
int DepIndex = OrdereredToDo.IndexOf(Dep);
if (DepIndex >= MyIndex)
{
throw new AutomationException("Topological sort error, node {0} has a pseduodependency of {1} which sorted after it.", NodeToDo, Dep);
}
}
}
}
string FakeFail = ParseParamValue("FakeFail");
if (CommanderSetup)
{
if (OrdereredToDo.Count == 0)
{
throw new AutomationException("No nodes to do!");
}
var ECProps = new List<string>();
ECProps.Add(String.Format("TimeIndex={0}", TimeIndex));
foreach (var NodePair in FullNodeList)
{
ECProps.Add(string.Format("AllNodes/{0}={1}", NodePair.Key, NodePair.Value));
}
foreach (var NodePair in FullNodeDirectDependencies)
{
ECProps.Add(string.Format("DirectDependencies/{0}={1}", NodePair.Key, NodePair.Value));
}
foreach (var NodePair in FullNodeListSortKey)
{
ECProps.Add(string.Format("SortKey/{0}={1}", NodePair.Key, NodePair.Value));
}
foreach (var NodePair in FullNodeDependedOnBy)
{
ECProps.Add(string.Format("DependedOnBy/{0}={1}", NodePair.Key, NodePair.Value));
}
foreach (var NodePair in FullNodeDependentPromotions)
{
ECProps.Add(string.Format("DependentPromotions/{0}={1}", NodePair.Key, NodePair.Value));
}
foreach (var Node in SeparatePromotables)
{
ECProps.Add(string.Format("PossiblePromotables/{0}={1}", Node, ""));
}
var ECJobProps = new List<string>();
if (ExplicitTrigger != "")
{
ECJobProps.Add("IsRoot=0");
}
else
{
ECJobProps.Add("IsRoot=1");
}
var FilteredOrdereredToDo = new List<string>();
// remove nodes that have unfinished triggers
foreach (var NodeToDo in OrdereredToDo)
{
string ControllingTrigger = GetControllingTrigger(NodeToDo);
bool bNoUnfinishedTriggers = !UnfinishedTriggers.Contains(ControllingTrigger);
if (bNoUnfinishedTriggers)
{
// if we are triggering, then remove nodes that are not controlled by the trigger or are dependencies of this trigger
if (!String.IsNullOrEmpty(ExplicitTrigger))
{
if (ExplicitTrigger != NodeToDo && !NodeDependsOn(NodeToDo, ExplicitTrigger) && !NodeDependsOn(ExplicitTrigger, NodeToDo))
{
continue; // this wasn't on the chain related to the trigger we are triggering, so it is not relevant
}
}
if (bPreflightBuild && !bSkipTriggers && GUBPNodes[NodeToDo].TriggerNode())
{
// in preflight builds, we are either skipping triggers (and running things downstream) or we just stop at triggers and don't make them available for triggering.
continue;
}
FilteredOrdereredToDo.Add(NodeToDo);
}
}
OrdereredToDo = FilteredOrdereredToDo;
Log("*********** EC Nodes, in order.");
PrintNodes(this, OrdereredToDo, LocalOnly, UnfinishedTriggers);
// here we are just making sure everything before the explicit trigger is completed.
if (!String.IsNullOrEmpty(ExplicitTrigger))
{
foreach (var NodeToDo in FilteredOrdereredToDo)
{
if (GUBPNodes[NodeToDo].RunInEC() && !NodeIsAlreadyComplete(NodeToDo, LocalOnly) && NodeToDo != ExplicitTrigger && !NodeDependsOn(ExplicitTrigger, NodeToDo)) // if something is already finished, we don't put it into EC
{
throw new AutomationException("We are being asked to process node {0}, however, this is an explicit trigger {1}, so everything before it should already be handled. It seems likely that you waited too long to run the trigger. You will have to do a new build from scratch.", NodeToDo, ExplicitTrigger);
}
}
}
string LastSticky = "";
bool HitNonSticky = false;
bool bHaveECNodes = false;
List<string> StepList = new List<string>();
StepList.Add("use strict;");
StepList.Add("use diagnostics;");
StepList.Add("use ElectricCommander();");
StepList.Add("my $ec = new ElectricCommander;");
StepList.Add("$ec->setTimeout(600);");
StepList.Add("my $batch = $ec->newBatch(\"serial\");");
// sticky nodes are ones that we run on the main agent. We run then first and they must not be intermixed with parallel jobs
foreach (var NodeToDo in OrdereredToDo)
{
if (GUBPNodes[NodeToDo].RunInEC() && !NodeIsAlreadyComplete(NodeToDo, LocalOnly)) // if something is already finished, we don't put it into EC
{
bHaveECNodes = true;
if (GUBPNodes[NodeToDo].IsSticky())
{
LastSticky = NodeToDo;
if (HitNonSticky && !bSkipTriggers)
{
throw new AutomationException("Sticky and non-sticky jobs did not sort right.");
}
}
else
{
HitNonSticky = true;
}
}
}
string ParentPath = ParseParamValue("ParentPath");
string BaseArgs = String.Format("$batch->createJobStep({{parentPath => '{0}'", ParentPath);
bool bHasNoop = false;
if (LastSticky == "" && bHaveECNodes)
{
// if we don't have any sticky nodes and we have other nodes, we run a fake noop just to release the resource
string Args = String.Format("{0}, subprocedure => 'GUBP_UAT_Node', parallel => '0', jobStepName => 'Noop', actualParameter => [{{actualParameterName => 'NodeName', value => 'Noop'}}, {{actualParameterName => 'Sticky', value =>'1' }}], releaseMode => 'release'}});", BaseArgs);
StepList.Add(Args);
bHasNoop = true;
}
var FakeECArgs = new List<string>();
var AgentGroupChains = new Dictionary<string, List<string>>();
foreach (var NodeToDo in OrdereredToDo)
{
if (GUBPNodes[NodeToDo].RunInEC() && !NodeIsAlreadyComplete(NodeToDo, LocalOnly)) // if something is already finished, we don't put it into EC
{
string MyAgentGroup = GUBPNodes[NodeToDo].AgentSharingGroup;
if (MyAgentGroup != "")
{
if (!AgentGroupChains.ContainsKey(MyAgentGroup))
{
AgentGroupChains.Add(MyAgentGroup, new List<string>{NodeToDo});
}
else
{
AgentGroupChains[MyAgentGroup].Add(NodeToDo);
}
}
}
}
foreach (var NodeToDo in OrdereredToDo)
{
if (GUBPNodes[NodeToDo].RunInEC() && !NodeIsAlreadyComplete(NodeToDo, LocalOnly)) // if something is already finished, we don't put it into EC
{
string EMails;
var NodeProps = GetECPropsForNode(NodeToDo, CLString, out EMails);
ECProps.AddRange(NodeProps);
bool Sticky = GUBPNodes[NodeToDo].IsSticky();
bool DoParallel = !Sticky;
if (Sticky && GUBPNodes[NodeToDo].ECAgentString() != "")
{
throw new AutomationException("Node {1} is sticky but has agent requirements.", NodeToDo);
}
string Procedure = GUBPNodes[NodeToDo].ECProcedure();
if(GUBPNodes[NodeToDo].IsSticky() && NodeToDo == LastSticky)
{
Procedure = Procedure + "_Release";
}
string Args = String.Format("{0}, subprocedure => '{1}', parallel => '{2}', jobStepName => '{3}', actualParameter => [{{actualParameterName => 'NodeName', value =>'{4}'}}",
BaseArgs, Procedure, DoParallel ? 1 : 0, NodeToDo, NodeToDo);
string ProcedureParams = GUBPNodes[NodeToDo].ECProcedureParams();
if (!String.IsNullOrEmpty(ProcedureParams))
{
Args = Args + ProcedureParams;
}
if ((Procedure == "GUBP_UAT_Trigger" || Procedure == "GUBP_Hardcoded_Trigger") && !String.IsNullOrEmpty(EMails))
{
Args = Args + ", {actualParameterName => 'EmailsForTrigger', value => \'" + EMails + "\'}";
}
Args = Args + "]";
string PreCondition = "";
string RunCondition = "";
var UncompletedEcDeps = new List<string>();
var UncompletedCompletedDeps = new List<string>();
string MyAgentGroup = GUBPNodes[NodeToDo].AgentSharingGroup;
bool bDoNestedJobstep = false;
bool bDoFirstNestedJobstep = false;
string NodeParentPath = ParentPath;
string PreconditionParentPath;
if (GUBPNodes[NodeToDo].GetFullName().Contains("MakeBuild") && GUBPNodes[NodeToDo].FullNamesOfPseudosependencies.Contains(WaitForFormalUserInput.StaticGetFullName()) && !bGraphSubset)
{
RemovePseudodependencyFromNode(NodeToDo, WaitForFormalUserInput.StaticGetFullName());
PreconditionParentPath = GetPropertyFromStep("/myWorkflow/ParentJob");
UncompletedEcDeps = GetECDependencies(NodeToDo);
}
else
{
PreconditionParentPath = ParentPath;
var EcDeps = GetECDependencies(NodeToDo);
foreach (var Dep in EcDeps)
{
if (GUBPNodes[Dep].RunInEC() && !NodeIsAlreadyComplete(Dep, LocalOnly) && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC
{
if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(NodeToDo))
{
throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo, Dep);
}
UncompletedEcDeps.Add(Dep);
}
}
}
var PreConditionUncompletedEcDeps = UncompletedEcDeps;
var CompletedDeps = GetCompletedOnlyDependencies(NodeToDo);
foreach (var Dep in CompletedDeps)
{
if (GUBPNodes[Dep].RunInEC() && !NodeIsAlreadyComplete(Dep, LocalOnly) && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC
{
if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(NodeToDo))
{
throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", NodeToDo, Dep);
}
UncompletedCompletedDeps.Add(Dep);
}
}
if (MyAgentGroup != "")
{
bDoNestedJobstep = true;
NodeParentPath = ParentPath + "/jobSteps[" + MyAgentGroup + "]";
PreConditionUncompletedEcDeps = new List<string>();
var MyChain = AgentGroupChains[MyAgentGroup];
int MyIndex = MyChain.IndexOf(NodeToDo);
if (MyIndex > 0)
{
PreConditionUncompletedEcDeps.Add(MyChain[MyIndex - 1]);
}
else
{
bDoFirstNestedJobstep = bDoNestedJobstep;
// to avoid idle agents (and also EC doesn't actually reserve our agent!), we promote all dependencies to the first one
foreach (var Chain in MyChain)
{
var EcDeps = GetECDependencies(Chain);
foreach (var Dep in EcDeps)
{
if (GUBPNodes[Dep].RunInEC() && !NodeIsAlreadyComplete(Dep, LocalOnly) && OrdereredToDo.Contains(Dep)) // if something is already finished, we don't put it into EC
{
if (OrdereredToDo.IndexOf(Dep) > OrdereredToDo.IndexOf(Chain))
{
throw new AutomationException("Topological sort error, node {0} has a dependency of {1} which sorted after it.", Chain, Dep);
}
if (!MyChain.Contains(Dep) && !PreConditionUncompletedEcDeps.Contains(Dep))
{
PreConditionUncompletedEcDeps.Add(Dep);
}
}
}
}
}
}
if (bHasNoop && PreConditionUncompletedEcDeps.Count == 0)
{
PreConditionUncompletedEcDeps.Add("Noop");
}
if (PreConditionUncompletedEcDeps.Count > 0)
{
PreCondition = "\"\\$\" . \"[/javascript if(";
// these run "parallel", but we add preconditions to serialize them
int Index = 0;
foreach (var Dep in PreConditionUncompletedEcDeps)
{
PreCondition = PreCondition + "getProperty('" + GetJobStep(PreconditionParentPath, Dep) + "/status\') == \'completed\'";
Index++;
if (Index != PreConditionUncompletedEcDeps.Count)
{
PreCondition = PreCondition + " && ";
}
}
PreCondition = PreCondition + ") true;]\"";
}
if(UncompletedCompletedDeps.Count > 0)
{
PreCondition = "\"\\$\" . \"[/javascript if(";
// these run "parallel", but we add preconditions to serialize them
int Index = 0;
foreach (var Dep in CompletedDeps)
{
if (GUBPNodes[Dep].RunInEC())
{
PreCondition = PreCondition + "getProperty('" + GetJobStep(PreconditionParentPath, Dep) + "/status\') == \'completed\'";
Index++;
if (Index != CompletedDeps.Count)
{
PreCondition = PreCondition + " && ";
}
}
}
PreCondition = PreCondition + ") true;]\"";
}
if (UncompletedEcDeps.Count > 0)
{
RunCondition = "\"\\$\" . \"[/javascript if(";
int Index = 0;
foreach (var Dep in UncompletedEcDeps)
{
RunCondition = RunCondition + "((\'\\$\" . \"[" + GetJobStep(PreconditionParentPath, Dep) + "/outcome]\' == \'success\') || ";
RunCondition = RunCondition + "(\'\\$\" . \"[" + GetJobStep(PreconditionParentPath, Dep) + "/outcome]\' == \'warning\'))";
Index++;
if (Index != UncompletedEcDeps.Count)
{
RunCondition = RunCondition + " && ";
}
}
RunCondition = RunCondition + ")true; else false;]\"";
}
if (bDoNestedJobstep)
{
if (bDoFirstNestedJobstep)
{
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}', jobStepName => '{1}', parallel => '1'",
ParentPath, MyAgentGroup);
if (!String.IsNullOrEmpty(PreCondition))
{
NestArgs = NestArgs + ", precondition => " + PreCondition;
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetPool', subprocedure => 'GUBP{3}_AgentShare_GetPool', parallel => '1', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{4}'}}, {{actualParameterName => 'NodeName', value => '{5}'}}]",
ParentPath, MyAgentGroup, MyAgentGroup, GUBPNodes[NodeToDo].ECProcedureInfix(), MyAgentGroup, NodeToDo);
if (!String.IsNullOrEmpty(PreCondition))
{
NestArgs = NestArgs + ", precondition => " + PreCondition;
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
string NestArgs = String.Format("$batch->createJobStep({{parentPath => '{0}/jobSteps[{1}]', jobStepName => '{2}_GetAgent', subprocedure => 'GUBP{3}_AgentShare_GetAgent', parallel => '1', exclusiveMode => 'call', resourceName => '{4}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', value => '{5}'}}, {{actualParameterName => 'NodeName', value=> '{6}'}}]",
ParentPath, MyAgentGroup, MyAgentGroup, GUBPNodes[NodeToDo].ECProcedureInfix(),
String.Format("$[/myJob/jobSteps[{0}]/ResourcePool]", MyAgentGroup),
MyAgentGroup, NodeToDo);
{
NestArgs = NestArgs + ", precondition => ";
NestArgs = NestArgs + "\"\\$\" . \"[/javascript if(";
NestArgs = NestArgs + "getProperty('" + PreconditionParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetPool]/status') == 'completed'";
NestArgs = NestArgs + ") true;]\"";
}
NestArgs = NestArgs + "});";
StepList.Add(NestArgs);
}
{
PreCondition = "\"\\$\" . \"[/javascript if(";
PreCondition = PreCondition + "getProperty('" + PreconditionParentPath + "/jobSteps[" + MyAgentGroup + "]/jobSteps[" + MyAgentGroup + "_GetAgent]/status') == 'completed'";
PreCondition = PreCondition + ") true;]\"";
}
}
Args = Args.Replace(String.Format("parentPath => '{0}'", ParentPath), String.Format("parentPath => '{0}'", NodeParentPath));
Args = Args.Replace("UAT_Node_Parallel_AgentShare", "UAT_Node_Parallel_AgentShare3");
}
if (!String.IsNullOrEmpty(PreCondition))
{
Args = Args + ", precondition => " + PreCondition;
}
if (!String.IsNullOrEmpty(RunCondition))
{
Args = Args + ", condition => " + RunCondition;
}
#if false
// this doesn't work because it includes precondition time
if (GUBPNodes[NodeToDo].TimeoutInMinutes() > 0)
{
Args = Args + String.Format(" --timeLimitUnits minutes --timeLimit {0}", GUBPNodes[NodeToDo].TimeoutInMinutes());
}
#endif
if (Sticky && NodeToDo == LastSticky)
{
Args = Args + ", releaseMode => 'release'";
}
Args = Args + "});";
StepList.Add(Args);
if (bFakeEC &&
!UnfinishedTriggers.Contains(NodeToDo) &&
(GUBPNodes[NodeToDo].ECProcedure().StartsWith("GUBP_UAT_Node") || GUBPNodes[NodeToDo].ECProcedure().StartsWith("GUBP_Mac_UAT_Node")) // other things we really can't test
) // unfinished triggers are never run directly by EC, rather it does another job setup
{
string Arg = String.Format("gubp -Node={0} -FakeEC {1} {2} {3} {4} {5}",
NodeToDo,
bFake ? "-Fake" : "" ,
ParseParam("AllPlatforms") ? "-AllPlatforms" : "",
ParseParam("UnfinishedTriggersFirst") ? "-UnfinishedTriggersFirst" : "",
ParseParam("UnfinishedTriggersParallel") ? "-UnfinishedTriggersParallel" : "",
ParseParam("WithMac") ? "-WithMac" : ""
);
string Node = ParseParamValue("-Node");
if (!String.IsNullOrEmpty(Node))
{
Arg = Arg + " -Node=" + Node;
}
if (!String.IsNullOrEmpty(FakeFail))
{
Arg = Arg + " -FakeFail=" + FakeFail;
}
FakeECArgs.Add(Arg);
}
if (MyAgentGroup != "" && !bDoNestedJobstep)
{
var MyChain = AgentGroupChains[MyAgentGroup];
int MyIndex = MyChain.IndexOf(NodeToDo);
if (MyIndex == MyChain.Count - 1)
{
var RelPreCondition = "\"\\$\" . \"[/javascript if(";
// this runs "parallel", but we a precondition to serialize it
RelPreCondition = RelPreCondition + "getProperty('" + PreconditionParentPath + "/jobSteps[" + NodeToDo + "]/status') == 'completed'";
RelPreCondition = RelPreCondition + ") true;]\"";
// we need to release the resource
string RelArgs = String.Format("{0}, subprocedure => 'GUBP_Release_AgentShare', parallel => '1', jobStepName => 'Release_{1}', actualParameter => [{{actualParameterName => 'AgentSharingGroup', valued => '{2}'}}], releaseMode => 'release', precondition => '{3}'",
BaseArgs, MyAgentGroup, MyAgentGroup, RelPreCondition);
StepList.Add(RelArgs);
}
}
}
}
WriteECPerl(StepList);
RunECTool(String.Format("setProperty \"/myWorkflow/HasTests\" \"{0}\"", bHasTests));
{
ECProps.Add("GUBP_LoadedProps=1");
string BranchDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchDef.properties");
CommandUtils.WriteAllLines(BranchDefFile, ECProps.ToArray());
RunECTool(String.Format("setProperty \"/myWorkflow/BranchDefFile\" \"{0}\"", BranchDefFile.Replace("\\", "\\\\")));
}
{
ECProps.Add("GUBP_LoadedJobProps=1");
string BranchJobDefFile = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LogFolder, "BranchJobDef.properties");
CommandUtils.WriteAllLines(BranchJobDefFile, ECProps.ToArray());
RunECTool(String.Format("setProperty \"/myJob/BranchJobDefFile\" \"{0}\"", BranchJobDefFile.Replace("\\", "\\\\")));
}
if (bFakeEC)
{
foreach (var Args in FakeECArgs)
{
RunUAT(CmdEnv, Args);
}
}
Log("Commander setup only, done.");
PrintRunTime();
return;
}
if (ParseParam("SaveGraph"))
{
SaveGraphVisualization(OrdereredToDo);
}
if (bListOnly)
{
Log("List only, done.");
return;
}
var BuildProductToNodeMap = new Dictionary<string, string>();
foreach (var NodeToDo in OrdereredToDo)
{
if (GUBPNodes[NodeToDo].BuildProducts != null || GUBPNodes[NodeToDo].AllDependencyBuildProducts != null)
{
throw new AutomationException("topological sort error");
}
GUBPNodes[NodeToDo].AllDependencyBuildProducts = new List<string>();
GUBPNodes[NodeToDo].AllDependencies = new List<string>();
foreach (var Dep in GUBPNodes[NodeToDo].FullNamesOfDependencies)
{
GUBPNodes[NodeToDo].AddAllDependent(Dep);
if (GUBPNodes[Dep].AllDependencies == null)
{
if (!bOnlyNode)
{
throw new AutomationException("Node {0} was not processed yet3? Processing {1}", Dep, NodeToDo);
}
}
else
{
foreach (var DepDep in GUBPNodes[Dep].AllDependencies)
{
GUBPNodes[NodeToDo].AddAllDependent(DepDep);
}
}
if (GUBPNodes[Dep].BuildProducts == null)
{
if (!bOnlyNode)
{
throw new AutomationException("Node {0} was not processed yet? Processing {1}", Dep, NodeToDo);
}
}
else
{
foreach (var Prod in GUBPNodes[Dep].BuildProducts)
{
GUBPNodes[NodeToDo].AddDependentBuildProduct(Prod);
}
if (GUBPNodes[Dep].AllDependencyBuildProducts == null)
{
throw new AutomationException("Node {0} was not processed yet2? Processing {1}", Dep, NodeToDo);
}
foreach (var Prod in GUBPNodes[Dep].AllDependencyBuildProducts)
{
GUBPNodes[NodeToDo].AddDependentBuildProduct(Prod);
}
}
}
string NodeStoreName = StoreName + "-" + GUBPNodes[NodeToDo].GetFullName();
string GameNameIfAny = GUBPNodes[NodeToDo].GameNameIfAnyForTempStorage();
string StorageRootIfAny = GUBPNodes[NodeToDo].RootIfAnyForTempStorage();
if (bFake)
{
StorageRootIfAny = ""; // we don't rebase fake runs since those are entirely "records of success", which are always in the logs folder
}
// this is kinda complicated
bool SaveSuccessRecords = (IsBuildMachine || bFakeEC) && // no real reason to make these locally except for fakeEC tests
(!GUBPNodes[NodeToDo].TriggerNode() || GUBPNodes[NodeToDo].IsSticky()) // trigger nodes are run twice, one to start the new workflow and once when it is actually triggered, we will save reconds for the latter
&& (GUBPNodes[NodeToDo].RunInEC() || !GUBPNodes[NodeToDo].IsAggregate()); //aggregates not in EC can be "run" multiple times, so we can't track those
Log("***** Running GUBP Node {0} -> {1} : {2}", GUBPNodes[NodeToDo].GetFullName(), GameNameIfAny, NodeStoreName);
if (NodeIsAlreadyComplete(NodeToDo, LocalOnly))
{
if (NodeToDo == VersionFilesNode.StaticGetFullName() && !IsBuildMachine)
{
Log("***** NOT ****** Retrieving GUBP Node {0} from {1}; it is the version files.", GUBPNodes[NodeToDo].GetFullName(), NodeStoreName);
GUBPNodes[NodeToDo].BuildProducts = new List<string>();
}
else
{
Log("***** Retrieving GUBP Node {0} from {1}", GUBPNodes[NodeToDo].GetFullName(), NodeStoreName);
bool WasLocal;
try
{
GUBPNodes[NodeToDo].BuildProducts = RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, GameNameIfAny, StorageRootIfAny);
}
catch
{
if(GameNameIfAny != "")
{
GUBPNodes[NodeToDo].BuildProducts = RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, "", StorageRootIfAny);
}
else
{
throw new AutomationException("Build Products cannot be found for node {0}", NodeToDo);
}
}
if (!WasLocal)
{
GUBPNodes[NodeToDo].PostLoadFromSharedTempStorage(this);
}
}
}
else
{
if (SaveSuccessRecords)
{
SaveStatus(NodeToDo, StartedTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny);
}
var BuildDuration = 0.0;
try
{
if (!String.IsNullOrEmpty(FakeFail) && FakeFail.Equals(NodeToDo, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Failing node {0} by request.", NodeToDo);
}
if (bFake)
{
Log("***** FAKE!! Building GUBP Node {0} for {1}", NodeToDo, NodeStoreName);
GUBPNodes[NodeToDo].DoFakeBuild(this);
}
else
{
Log("***** Building GUBP Node {0} for {1}", NodeToDo, NodeStoreName);
var StartTime = DateTime.UtcNow;
GUBPNodes[NodeToDo].DoBuild(this);
BuildDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000;
}
if (!GUBPNodes[NodeToDo].IsAggregate())
{
var StoreDuration = 0.0;
var StartTime = DateTime.UtcNow;
StoreToTempStorage(CmdEnv, NodeStoreName, GUBPNodes[NodeToDo].BuildProducts, !bSaveSharedTempStorage, GameNameIfAny, StorageRootIfAny);
StoreDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000;
Log("Took {0} seconds to store build products", StoreDuration);
if (IsBuildMachine)
{
RunECTool(String.Format("setProperty \"/myJobStep/StoreDuration\" \"{0}\"", StoreDuration.ToString()));
}
if (ParseParam("StompCheck"))
{
foreach (var Dep in GUBPNodes[NodeToDo].AllDependencies)
{
try
{
bool WasLocal;
RetrieveFromTempStorage(CmdEnv, NodeStoreName, out WasLocal, GameNameIfAny, StorageRootIfAny);
if (!WasLocal)
{
throw new AutomationException("Retrieve was not local?");
}
}
catch(Exception Ex)
{
throw new AutomationException("Node {0} stomped Node {1} Ex: {2}", NodeToDo, Dep, LogUtils.FormatException(Ex));
}
}
}
}
}
catch (Exception Ex)
{
if (SaveSuccessRecords)
{
UpdateNodeHistory(NodeToDo, CLString);
SaveStatus(NodeToDo, FailedTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny, ParseParamValue("MyJobStepId"));
UpdateECProps(NodeToDo, CLString);
if (IsBuildMachine)
{
GetFailureEmails(NodeToDo, CLString);
}
UpdateECBuildTime(NodeToDo, BuildDuration);
}
Log("{0}", ExceptionToString(Ex));
if (GUBPNodesHistory.ContainsKey(NodeToDo))
{
var History = GUBPNodesHistory[NodeToDo];
Log("Changes since last green *********************************");
Log("");
Log("");
Log("");
PrintDetailedChanges(History);
Log("End changes since last green");
}
string FailInfo = "";
FailInfo += "********************************* Main log file";
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail();
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
string OtherLog = "See logfile for details: '";
if (FailInfo.Contains(OtherLog))
{
string LogFile = FailInfo.Substring(FailInfo.IndexOf(OtherLog) + OtherLog.Length);
if (LogFile.Contains("'"))
{
LogFile = CombinePaths(CmdEnv.LogFolder, LogFile.Substring(0, LogFile.IndexOf("'")));
if (FileExists_NoExceptions(LogFile))
{
FailInfo += "********************************* Sub log file " + LogFile;
FailInfo += Environment.NewLine + Environment.NewLine;
FailInfo += LogUtils.GetLogTail(LogFile);
FailInfo += Environment.NewLine + Environment.NewLine + Environment.NewLine;
}
}
}
string Filename = CombinePaths(CmdEnv.LogFolder, "LogTailsAndChanges.log");
WriteAllText(Filename, FailInfo);
throw(Ex);
}
if (SaveSuccessRecords)
{
UpdateNodeHistory(NodeToDo, CLString);
SaveStatus(NodeToDo, SucceededTempStorageSuffix, NodeStoreName, bSaveSharedTempStorage, GameNameIfAny);
UpdateECProps(NodeToDo, CLString);
if (IsBuildMachine)
{
GetFailureEmails(NodeToDo, CLString);
}
UpdateECBuildTime(NodeToDo, BuildDuration);
}
}
foreach (var Product in GUBPNodes[NodeToDo].BuildProducts)
{
if (BuildProductToNodeMap.ContainsKey(Product))
{
throw new AutomationException("Overlapping build product: {0} and {1} both produce {2}", BuildProductToNodeMap[Product], NodeToDo, Product);
}
BuildProductToNodeMap.Add(Product, NodeToDo);
}
}
PrintRunTime();
}